From 7ac32a8bf16ea0bc3ac14413e4207d836325bbe1 Mon Sep 17 00:00:00 2001 From: yokowu <18836617@qq.com> Date: Fri, 8 Aug 2025 17:30:18 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=AE=9E=E7=8E=B0=E6=89=AB=E6=8F=8F?= =?UTF-8?q?=E5=99=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/backend-ci-cd.yml | 38 ++++++++++++- backend/Makefile | 14 +++++ backend/build/Dockerfile | 10 +--- backend/build/Dockerfile.scanner | 32 +++++++++++ backend/cmd/scanner/main.go | 55 +++++++++++++++++++ backend/cmd/scanner/wire.go | 32 +++++++++++ backend/cmd/scanner/wire_gen.go | 49 +++++++++++++++++ backend/domain/security.go | 7 +++ backend/internal/proxy/usecase/proxy.go | 21 +++++-- .../scanner/handler/http/v1/scanner.go | 38 +++++++++++++ 10 files changed, 280 insertions(+), 16 deletions(-) create mode 100644 backend/build/Dockerfile.scanner create mode 100644 backend/cmd/scanner/main.go create mode 100644 backend/cmd/scanner/wire.go create mode 100644 backend/cmd/scanner/wire_gen.go create mode 100644 backend/internal/scanner/handler/http/v1/scanner.go diff --git a/.github/workflows/backend-ci-cd.yml b/.github/workflows/backend-ci-cd.yml index dd7fd24..2c23bfb 100644 --- a/.github/workflows/backend-ci-cd.yml +++ b/.github/workflows/backend-ci-cd.yml @@ -160,6 +160,26 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max + - name: Build and push scanner image + uses: docker/build-push-action@v5 + with: + context: ./backend + file: ./backend/build/Dockerfile.scanner + push: true + platforms: ${{ matrix.platform }} + tags: | + ${{ env.REGISTRY }}/scanner:${{ steps.get_version.outputs.VERSION }}-${{ matrix.arch }} + ${{ env.REGISTRY }}/scanner:latest-${{ matrix.arch }} + build-args: | + GOCACHE=/tmp/go-build + GOMODCACHE=/tmp/go-mod + REPO_COMMIT=${{ github.sha }} + VERSION=${{ steps.get_version.outputs.VERSION }} + BUILD_TIME=${{ steps.get_build_time.outputs.BUILD_TIME }} + GIT_COMMIT=${{ steps.get_git_commit.outputs.GIT_COMMIT }} + cache-from: type=gha + cache-to: type=gha,mode=max + create-manifest: needs: build runs-on: ubuntu-latest @@ -176,14 +196,26 @@ jobs: run: | VERSION=${{ needs.build.outputs.version }} - # Create and push version manifest + # Create and push backend version manifest docker manifest create ${{ env.REGISTRY }}/backend:${VERSION} \ ${{ env.REGISTRY }}/backend:${VERSION}-x86_64 \ ${{ env.REGISTRY }}/backend:${VERSION}-aarch64 docker manifest push ${{ env.REGISTRY }}/backend:${VERSION} - # Create and push latest manifest + # Create and push backend latest manifest docker manifest create ${{ env.REGISTRY }}/backend:latest \ ${{ env.REGISTRY }}/backend:latest-x86_64 \ ${{ env.REGISTRY }}/backend:latest-aarch64 - docker manifest push ${{ env.REGISTRY }}/backend:latest \ No newline at end of file + docker manifest push ${{ env.REGISTRY }}/backend:latest + + # Create and push scanner version manifest + docker manifest create ${{ env.REGISTRY }}/scanner:${VERSION} \ + ${{ env.REGISTRY }}/scanner:${VERSION}-x86_64 \ + ${{ env.REGISTRY }}/scanner:${VERSION}-aarch64 + docker manifest push ${{ env.REGISTRY }}/scanner:${VERSION} + + # Create and push scanner latest manifest + docker manifest create ${{ env.REGISTRY }}/scanner:latest \ + ${{ env.REGISTRY }}/scanner:latest-x86_64 \ + ${{ env.REGISTRY }}/scanner:latest-aarch64 + docker manifest push ${{ env.REGISTRY }}/scanner:latest \ No newline at end of file diff --git a/backend/Makefile b/backend/Makefile index 8c66c22..45a13a6 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -23,6 +23,20 @@ image: --output ${OUTPUT} \ . +image-scanner: + docker buildx build \ + -f build/Dockerfile.scanner \ + --build-arg GOCACHE=${GOCACHE} \ + --build-arg GOMODCACHE=${GOMODCACHE} \ + --build-arg REPO_COMMIT=$(shell git rev-parse HEAD) \ + --build-arg VERSION=${VERSION} \ + --build-arg BUILD_TIME=${BUILD_TIME} \ + --build-arg GIT_COMMIT=${GIT_COMMIT} \ + --platform ${PLATFORM} \ + --tag ${REGISTRY}/scanner:${TAG} \ + --output ${OUTPUT} \ + . + wire: wire pro/cmd/server/wire.go pro/cmd/server/main.go diff --git a/backend/build/Dockerfile b/backend/build/Dockerfile index fc30dd7..733b119 100644 --- a/backend/build/Dockerfile +++ b/backend/build/Dockerfile @@ -21,18 +21,12 @@ go build \ -o /out/main \ pro/cmd/server/main.go pro/cmd/server/wire_gen.go -FROM debian:bullseye-20250721-slim as binary - -RUN apt-get update && \ - apt-get install -y ca-certificates && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* && \ - update-ca-certificates +FROM alpine:3.22.1 as binary WORKDIR /app ADD migration ./migration -ADD assets ./assets +ADD assets/vsix ./assets/vsix COPY --from=builder /out/main /app/main diff --git a/backend/build/Dockerfile.scanner b/backend/build/Dockerfile.scanner new file mode 100644 index 0000000..81592cf --- /dev/null +++ b/backend/build/Dockerfile.scanner @@ -0,0 +1,32 @@ +FROM --platform=$BUILDPLATFORM golang:1.23-alpine AS builder + +WORKDIR /src +ENV CGO_ENABLED=0 + +COPY go.* . +ARG GOMODCACHE +RUN --mount=type=cache,target=${GOMODCACHE} \ + go mod download + +ARG TARGETOS TARGETARCH GOCACHE +ARG VERSION +ARG BUILD_TIME +ARG GIT_COMMIT +RUN --mount=type=bind,target=. \ +--mount=type=cache,target=${GOMODCACHE} \ +--mount=type=cache,target=${GOCACHE} \ +GOOS=$TARGETOS GOARCH=$TARGETARCH \ +go build \ +-ldflags "-w -s -X 'github.com/chaitin/MonkeyCode/backend/pkg/version.Version=${VERSION}' -X 'github.com/chaitin/MonkeyCode/backend/pkg/version.BuildTime=${BUILD_TIME}' -X 'github.com/chaitin/MonkeyCode/backend/pkg/version.GitCommit=${GIT_COMMIT}'" \ +-o /out/main \ +cmd/scanner/main.go cmd/scanner/wire_gen.go + +FROM debian:bullseye-20250721-slim as binary + +WORKDIR /app + +ADD assets/sgp ./assets/sgp + +COPY --from=builder /out/main /app/main + +CMD [ "./main" ] \ No newline at end of file diff --git a/backend/cmd/scanner/main.go b/backend/cmd/scanner/main.go new file mode 100644 index 0000000..f7fc6e7 --- /dev/null +++ b/backend/cmd/scanner/main.go @@ -0,0 +1,55 @@ +package main + +import ( + "context" + + "github.com/google/wire" + + "github.com/chaitin/MonkeyCode/backend/config" + v1 "github.com/chaitin/MonkeyCode/backend/internal/scanner/handler/http/v1" + "github.com/chaitin/MonkeyCode/backend/pkg" + "github.com/chaitin/MonkeyCode/backend/pkg/logger" + "github.com/chaitin/MonkeyCode/backend/pkg/service" + "github.com/chaitin/MonkeyCode/backend/pkg/version" +) + +func main() { + s, err := newServer() + if err != nil { + panic(err) + } + + s.version.Print() + s.logger.With("config", s.config).Debug("config") + s.web.PrintRoutes() + + svc := service.NewService(service.WithPprof()) + svc.Add(s) + if err := svc.Run(); err != nil { + panic(err) + } +} + +// Name implements service.Servicer. +func (s *Server) Name() string { + return "Scanner Server" +} + +// Start implements service.Servicer. +func (s *Server) Start() error { + return s.web.Run(s.config.Server.Addr) +} + +// Stop implements service.Servicer. +func (s *Server) Stop() error { + return s.web.Echo().Shutdown(context.Background()) +} + +var AppSet = wire.NewSet( + wire.FieldsOf(new(*config.Config), "Logger"), + config.Init, + pkg.NewWeb, + logger.NewLogger, + version.NewVersionInfo, + v1.NewScannerHandler, +) diff --git a/backend/cmd/scanner/wire.go b/backend/cmd/scanner/wire.go new file mode 100644 index 0000000..e54b125 --- /dev/null +++ b/backend/cmd/scanner/wire.go @@ -0,0 +1,32 @@ +//go:build wireinject +// +build wireinject + +package main + +import ( + "log/slog" + + "github.com/google/wire" + + "github.com/GoYoko/web" + + "github.com/chaitin/MonkeyCode/backend/config" + v1 "github.com/chaitin/MonkeyCode/backend/internal/scanner/handler/http/v1" + "github.com/chaitin/MonkeyCode/backend/pkg/version" +) + +type Server struct { + config *config.Config + web *web.Web + logger *slog.Logger + version *version.VersionInfo + scanner *v1.ScannerHandler +} + +func newServer() (*Server, error) { + wire.Build( + wire.Struct(new(Server), "*"), + AppSet, + ) + return &Server{}, nil +} diff --git a/backend/cmd/scanner/wire_gen.go b/backend/cmd/scanner/wire_gen.go new file mode 100644 index 0000000..ad3a49d --- /dev/null +++ b/backend/cmd/scanner/wire_gen.go @@ -0,0 +1,49 @@ +// Code generated by Wire. DO NOT EDIT. + +//go:generate go run github.com/google/wire/cmd/wire +//go:build !wireinject +// +build !wireinject + +package main + +import ( + "github.com/GoYoko/web" + "github.com/chaitin/MonkeyCode/backend/config" + "github.com/chaitin/MonkeyCode/backend/internal/scanner/handler/http/v1" + "github.com/chaitin/MonkeyCode/backend/pkg" + "github.com/chaitin/MonkeyCode/backend/pkg/logger" + "github.com/chaitin/MonkeyCode/backend/pkg/version" + "log/slog" +) + +// Injectors from wire.go: + +func newServer() (*Server, error) { + configConfig, err := config.Init() + if err != nil { + return nil, err + } + web := pkg.NewWeb(configConfig) + loggerConfig := configConfig.Logger + slogLogger := logger.NewLogger(loggerConfig) + versionInfo := version.NewVersionInfo() + scannerHandler := v1.NewScannerHandler(web, slogLogger) + server := &Server{ + config: configConfig, + web: web, + logger: slogLogger, + version: versionInfo, + scanner: scannerHandler, + } + return server, nil +} + +// wire.go: + +type Server struct { + config *config.Config + web *web.Web + logger *slog.Logger + version *version.VersionInfo + scanner *v1.ScannerHandler +} diff --git a/backend/domain/security.go b/backend/domain/security.go index 3fe3e0c..abb0602 100644 --- a/backend/domain/security.go +++ b/backend/domain/security.go @@ -70,6 +70,13 @@ func (s *SecurityScanningBrief) From(e *db.SecurityScanning) *SecurityScanningBr return s } +type ScanReq struct { + TaskID string `json:"task_id"` + UserID string `json:"user_id"` + Workspace string `json:"workspace"` // 项目目录 + Language consts.SecurityScanningLanguage `json:"language"` // 扫描语言 +} + type CreateSecurityScanningReq struct { UserID string `json:"user_id"` Workspace string `json:"workspace"` // 项目目录 diff --git a/backend/internal/proxy/usecase/proxy.go b/backend/internal/proxy/usecase/proxy.go index b0f3b15..e837341 100644 --- a/backend/internal/proxy/usecase/proxy.go +++ b/backend/internal/proxy/usecase/proxy.go @@ -4,9 +4,9 @@ import ( "context" "fmt" "log/slog" + "net/http" "os" "path" - "strings" "time" "github.com/redis/go-redis/v9" @@ -38,7 +38,13 @@ func NewProxyUsecase( cfg *config.Config, redis *redis.Client, ) domain.ProxyUsecase { - client := request.NewClient("http", "monkeycode-scanner:8888", 15*time.Second) + client := request.NewClient("http", "monkeycode-scanner:8888", 30*time.Minute, request.WithTransport(&http.Transport{ + MaxIdleConns: 100, + MaxIdleConnsPerHost: 100, + MaxConnsPerHost: 100, + IdleConnTimeout: 90 * time.Second, + ForceAttemptHTTP2: true, + })) client.SetDebug(cfg.Debug) p := &ProxyUsecase{ repo: repo, @@ -128,7 +134,7 @@ func (p *ProxyUsecase) TaskHandle(ctx context.Context, task *queuerunner.Task[do p.logger.With("id", id).With("error", err).ErrorContext(ctx, "failed to get security scanning") return err } - prefix := fmt.Sprintf("/app/codes/%s", id) + prefix := fmt.Sprintf("/app/static/codes/%s", id) rootPath := path.Join(prefix, scanning.Edges.WorkspaceEdge.RootPath) defer os.RemoveAll(prefix) @@ -151,8 +157,13 @@ func (p *ProxyUsecase) TaskHandle(ctx context.Context, task *queuerunner.Task[do return err } - rule := strings.ToLower(string(scanning.Language)) - result, err := scan.Scan(task.ID, rootPath, rule) + result, err := request.Post[scan.Result](p.client, "/api/v1/scan", domain.ScanReq{ + TaskID: task.ID, + UserID: task.Data.UserID, + Workspace: rootPath, + Language: task.Data.Language, + }) + if err != nil { if err = p.securityRepo.Update(ctx, id, consts.SecurityScanningStatusFailed, &scan.Result{ Output: err.Error(), diff --git a/backend/internal/scanner/handler/http/v1/scanner.go b/backend/internal/scanner/handler/http/v1/scanner.go new file mode 100644 index 0000000..5352d95 --- /dev/null +++ b/backend/internal/scanner/handler/http/v1/scanner.go @@ -0,0 +1,38 @@ +package v1 + +import ( + "fmt" + "log/slog" + "net/http" + "strings" + + "github.com/GoYoko/web" + + "github.com/chaitin/MonkeyCode/backend/domain" + "github.com/chaitin/MonkeyCode/backend/pkg/scan" +) + +type ScannerHandler struct { + logger *slog.Logger +} + +func NewScannerHandler(w *web.Web, logger *slog.Logger) *ScannerHandler { + s := &ScannerHandler{ + logger: logger, + } + + w.POST("/api/v1/scan", web.BindHandler(s.Scan)) + + return s +} + +func (s *ScannerHandler) Scan(ctx *web.Context, req domain.ScanReq) error { + rule := strings.ToLower(string(req.Language)) + result, err := scan.Scan(req.TaskID, req.Workspace, rule) + if err != nil { + s.logger.With("id", req.TaskID).With("error", err).ErrorContext(ctx.Request().Context(), "failed to scan") + return fmt.Errorf("failed to scan: %w", err) + } + s.logger.With("id", req.TaskID).InfoContext(ctx.Request().Context(), "task done") + return ctx.JSON(http.StatusOK, result) +}