From 78e90e300d573bfd65bfe73011003499d910add3 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto <25837540+dwisiswant0@users.noreply.github.com> Date: Thu, 25 Dec 2025 02:35:44 +0700 Subject: [PATCH] ci: refactor workflows (#6728) Signed-off-by: Dwi Siswanto --- .github/workflows/auto-merge.yaml | 30 +++++------- .github/workflows/compat-checks.yaml | 4 +- .github/workflows/flamegraph.yaml | 61 ++++++++++++++++++++++++ .github/workflows/generate-docs.yaml | 4 +- .github/workflows/generate-pgo.yaml | 66 ++++++++++++-------------- .github/workflows/govulncheck.yaml | 2 +- .github/workflows/perf-regression.yaml | 4 +- .github/workflows/perf-test.yaml | 43 ----------------- .github/workflows/release.yaml | 18 +++++-- .github/workflows/tests.yaml | 39 ++------------- .goreleaser.yml | 5 +- 11 files changed, 132 insertions(+), 144 deletions(-) create mode 100644 .github/workflows/flamegraph.yaml delete mode 100644 .github/workflows/perf-test.yaml diff --git a/.github/workflows/auto-merge.yaml b/.github/workflows/auto-merge.yaml index f6bb2c5c2..8da0eb6e5 100644 --- a/.github/workflows/auto-merge.yaml +++ b/.github/workflows/auto-merge.yaml @@ -1,28 +1,24 @@ -name: πŸ€– Auto Merge +name: πŸ”€ Auto merge PR on: - pull_request_review: - types: [submitted] + # pull_request: + # types: [opened, synchronize, reopened, ready_for_review] + # pull_request_review: + # types: [submitted] workflow_run: - workflows: ["♾️ Compatibility Check"] - types: - - completed + workflows: ["♾️ Compatibility Checks"] + types: [completed] permissions: + contents: write pull-requests: write - issues: write - repository-projects: write jobs: - auto-merge: + auto-merge-dependabot: runs-on: ubuntu-latest - if: github.actor == 'dependabot[bot]' + if: ${{ github.event.workflow_run.conclusion == 'success' }} steps: - - uses: actions/checkout@v6 + - uses: projectdiscovery/actions/pr/approve@v1 + - uses: projectdiscovery/actions/pr/merge@v1 with: - token: ${{ secrets.DEPENDABOT_PAT }} - - - uses: ahmadnassri/action-dependabot-auto-merge@v2 - with: - github-token: ${{ secrets.DEPENDABOT_PAT }} - target: all \ No newline at end of file + auto: "true" \ No newline at end of file diff --git a/.github/workflows/compat-checks.yaml b/.github/workflows/compat-checks.yaml index 093bd6ba0..086bf6422 100644 --- a/.github/workflows/compat-checks.yaml +++ b/.github/workflows/compat-checks.yaml @@ -7,8 +7,8 @@ on: - dev jobs: - check: - if: github.actor == 'dependabot[bot]' + compat-checks: + if: ${{ github.actor == 'dependabot[bot]' }} runs-on: ubuntu-latest permissions: contents: write diff --git a/.github/workflows/flamegraph.yaml b/.github/workflows/flamegraph.yaml new file mode 100644 index 000000000..850407eb7 --- /dev/null +++ b/.github/workflows/flamegraph.yaml @@ -0,0 +1,61 @@ +name: πŸ“Š Flamegraph + +on: + workflow_call: {} + +jobs: + flamegraph: + name: "Flamegraph" + env: + PROFILE_MEM: "/tmp/nuclei-profile" + TARGET_URL: "http://honey.scanme.sh/-/?foo=bar" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: projectdiscovery/actions/setup/go@v1 + - uses: projectdiscovery/actions/cache/go-rod-browser@v1 + - uses: projectdiscovery/actions/cache/nuclei@v1 + - run: make build + + - name: "Setup environment (push)" + if: ${{ github.event_name == 'push' }} + run: | + echo "PROFILE_MEM=${PROFILE_MEM}-${GITHUB_REF_NAME}-${GITHUB_SHA}" >> $GITHUB_ENV + echo "FLAMEGRAPH_NAME=nuclei-${GITHUB_REF_NAME} (${GITHUB_SHA})" >> $GITHUB_ENV + - name: "Setup environment (pull_request)" + if: ${{ github.event_name == 'pull_request' }} + run: | + echo "PROFILE_MEM=${PROFILE_MEM}-pr-${{ github.event.number }}" >> $GITHUB_ENV + echo "FLAMEGRAPH_NAME=nuclei (PR #${{ github.event.number }})" >> $GITHUB_ENV + + - run: ./bin/nuclei -update-templates + - run: | + ./bin/nuclei -silent -target="${TARGET_URL}" \ + -disable-update-check \ + -no-mhe -no-httpx \ + -code -dast -file -headless \ + -enable-self-contained -enable-global-matchers \ + -rate-limit=0 \ + -concurrency=250 \ + -headless-concurrency=100 \ + -payload-concurrency=250 \ + -bulk-size=250 \ + -headless-bulk-size=100 \ + -profile-mem="${PROFILE_MEM}" + + - uses: projectdiscovery/actions/flamegraph@v1 + id: flamegraph-cpu + with: + profile: "${{ env.PROFILE_MEM }}.cpu" + name: "${{ env.FLAMEGRAPH_NAME }} CPU profiles" + continue-on-error: true + - uses: projectdiscovery/actions/flamegraph@v1 + id: flamegraph-mem + with: + profile: "${{ env.PROFILE_MEM }}.mem" + name: "${{ env.FLAMEGRAPH_NAME }} memory profiles" + continue-on-error: true + - if: ${{ steps.flamegraph-mem.outputs.message == '' }} + run: | + echo "::notice::CPU flamegraph: ${{ steps.flamegraph-cpu.outputs.url }}" + echo "::notice::Memory (heap) flamegraph: ${{ steps.flamegraph-mem.outputs.url }}" diff --git a/.github/workflows/generate-docs.yaml b/.github/workflows/generate-docs.yaml index 365ad32f9..d40b235c6 100644 --- a/.github/workflows/generate-docs.yaml +++ b/.github/workflows/generate-docs.yaml @@ -4,11 +4,11 @@ on: push: branches: - dev - workflow_dispatch: + workflow_dispatch: {} jobs: publish-docs: - if: "${{ !endsWith(github.actor, '[bot]') }}" + if: ${{ !endsWith(github.actor, '[bot]') }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 diff --git a/.github/workflows/generate-pgo.yaml b/.github/workflows/generate-pgo.yaml index 349306f2b..ae6ba4c5c 100644 --- a/.github/workflows/generate-pgo.yaml +++ b/.github/workflows/generate-pgo.yaml @@ -1,56 +1,50 @@ name: πŸ‘€ Generate PGO on: - push: - branches: ["dev"] - paths: - - '**.go' - - '**.mod' - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -# TODO(dwisiswant0): https://go.dev/doc/pgo#merging-profiles + workflow_dispatch: {} + workflow_call: {} jobs: pgo: - strategy: - matrix: - targets: [150] runs-on: ubuntu-latest if: github.repository == 'projectdiscovery/nuclei' permissions: contents: write env: PGO_FILE: "cmd/nuclei/default.pgo" - LIST_FILE: "/tmp/targets-${{ matrix.targets }}.txt" - PROFILE_MEM: "/tmp/nuclei-profile-${{ matrix.targets }}-targets" + TARGET: "https://honey.scanme.sh" + TARGET_COUNT: "100" + TARGET_LIST: "/tmp/targets.txt" + PROFILE_MEM: "/tmp/nuclei-profile" steps: - uses: actions/checkout@v6 - - uses: projectdiscovery/actions/setup/git@v1 - uses: projectdiscovery/actions/setup/go@v1 + - uses: projectdiscovery/actions/cache/go-rod-browser@v1 - uses: projectdiscovery/actions/cache/nuclei@v1 - - name: Generate list - run: for i in {1..${{ matrix.targets }}}; do echo "https://honey.scanme.sh/?_=${i}" >> "${LIST_FILE}"; done - # NOTE(dwisiswant0): use `-no-mhe` flag to get better samples. - - run: go run . -l "${LIST_FILE}" -profile-mem="${PROFILE_MEM}" -no-mhe - working-directory: cmd/nuclei/ - - run: mv "${PROFILE_MEM}.cpu" ${PGO_FILE} - # NOTE(dwisiswant0): shall we prune $PGO_FILE git history? - # if we prune it, this won't be linear since it requires a force-push. - # if we don't, the git objects will just keep growing bigger. - # - # Ref: - # - https://go.dev/blog/pgo#:~:text=We%20recommend%20committing%20default.pgo%20files%20to%20your%20repository - # - https://gist.github.com/nottrobin/5758221 - - uses: projectdiscovery/actions/commit@v1 - with: - files: "${PGO_FILE}" - message: "build: update PGO profile :robot:" - - run: git push origin $GITHUB_REF + - run: | + for i in {1..${{ env.TARGET_COUNT }}}; do + echo "${{ env.TARGET }}/-/?_=${i}" >> "${{ env.TARGET_LIST }}"; + done + - run: make build + - run: ./bin/nuclei -update-templates + - run: | + ./bin/nuclei -silent -list="${TARGET_LIST}" \ + -disable-update-check \ + -no-mhe -no-httpx \ + -code -dast -file -headless \ + -enable-self-contained -enable-global-matchers \ + -rate-limit=0 \ + -concurrency=250 \ + -headless-concurrency=100 \ + -payload-concurrency=250 \ + -bulk-size=250 \ + -headless-bulk-size=100 \ + -profile-mem="${PROFILE_MEM}" \ + env: + DISABLE_STDOUT: "1" + - run: mv "${PROFILE_MEM}.cpu" "${PGO_FILE}" - uses: actions/upload-artifact@v6 with: name: "pgo" path: "${{ env.PGO_FILE }}" + overwrite: true diff --git a/.github/workflows/govulncheck.yaml b/.github/workflows/govulncheck.yaml index 38edae248..24fc0919f 100644 --- a/.github/workflows/govulncheck.yaml +++ b/.github/workflows/govulncheck.yaml @@ -3,7 +3,7 @@ name: πŸ› govulncheck on: schedule: - cron: '0 0 * * 0' # Weekly - workflow_dispatch: + workflow_dispatch: {} jobs: govulncheck: diff --git a/.github/workflows/perf-regression.yaml b/.github/workflows/perf-regression.yaml index a44845def..b0f362a35 100644 --- a/.github/workflows/perf-regression.yaml +++ b/.github/workflows/perf-regression.yaml @@ -1,8 +1,8 @@ name: πŸ”¨ Performance Regression on: - workflow_call: - workflow_dispatch: + workflow_call: {} + workflow_dispatch: {} jobs: perf-regression: diff --git a/.github/workflows/perf-test.yaml b/.github/workflows/perf-test.yaml deleted file mode 100644 index ff40b824b..000000000 --- a/.github/workflows/perf-test.yaml +++ /dev/null @@ -1,43 +0,0 @@ -name: πŸ”¨ Performance Test - -on: - schedule: - - cron: '0 0 * * 0' # Weekly - workflow_dispatch: - -jobs: - perf-test: - strategy: - matrix: - count: [50, 100, 150] - runs-on: ubuntu-latest - if: github.repository == 'projectdiscovery/nuclei' - env: - LIST_FILE: "/tmp/targets-${{ matrix.count }}.txt" - PROFILE_MEM: "/tmp/nuclei-perf-test-${{ matrix.count }}" - steps: - - uses: actions/checkout@v6 - - uses: projectdiscovery/actions/setup/go@v1 - - run: make verify - - name: Generate list - run: for i in {1..${{ matrix.count }}}; do echo "https://honey.scanme.sh/?_=${i}" >> "${LIST_FILE}"; done - - run: go run . -l "${LIST_FILE}" -profile-mem="${PROFILE_MEM}" - env: - NUCLEI_ARGS: host-error-stats - working-directory: cmd/nuclei/ - - uses: projectdiscovery/actions/flamegraph@v1 - id: flamegraph-cpu - with: - profile: "${{ env.PROFILE_MEM }}.cpu" - name: "${{ env.FLAMEGRAPH_NAME }} CPU profiles" - continue-on-error: true - - uses: projectdiscovery/actions/flamegraph@v1 - id: flamegraph-mem - with: - profile: "${{ env.PROFILE_MEM }}.mem" - name: "${{ env.FLAMEGRAPH_NAME }} memory profiles" - continue-on-error: true - - if: ${{ steps.flamegraph-mem.outputs.message == '' }} - run: | - echo "::notice::CPU flamegraph: ${{ steps.flamegraph-cpu.outputs.url }}" - echo "::notice::Memory (heap) flamegraph: ${{ steps.flamegraph-mem.outputs.url }}" diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index acaecb596..84f5ba3a9 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -4,16 +4,28 @@ on: push: tags: - '*' - workflow_dispatch: + workflow_dispatch: {} -jobs: - release: +jobs: + pgo: + name: "Generate PGO" + uses: ./.github/workflows/generate-pgo.yaml + + release: + name: "Release" + needs: ["pgo"] runs-on: ubuntu-latest-16-cores steps: - uses: actions/checkout@v6 with: fetch-depth: 0 + - uses: actions/download-artifact@v6 + with: + name: "pgo" + path: "cmd/nuclei/" - uses: projectdiscovery/actions/setup/go@v1 + with: + go-version: "stable" - uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_USERNAME }} diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 42a46a67d..fd82e24d7 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -19,7 +19,7 @@ concurrency: jobs: lint: name: "Lint" - if: "${{ !endsWith(github.actor, '[bot]') }}" + if: ${{ !endsWith(github.actor, '[bot]') }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 @@ -157,42 +157,9 @@ jobs: flamegraph: name: "Flamegraph" needs: ["tests"] - env: - PROFILE_MEM: "/tmp/nuclei" - TARGET_URL: "http://scanme.sh/a/?b=c" - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - run: make build - - name: "Setup environment (push)" - if: ${{ github.event_name == 'push' }} - run: | - echo "PROFILE_MEM=${PROFILE_MEM}-${GITHUB_REF_NAME}-${GITHUB_SHA}" >> $GITHUB_ENV - echo "FLAMEGRAPH_NAME=nuclei-${GITHUB_REF_NAME} (${GITHUB_SHA})" >> $GITHUB_ENV - - name: "Setup environment (pull_request)" - if: ${{ github.event_name == 'pull_request' }} - run: | - echo "PROFILE_MEM=${PROFILE_MEM}-pr-${{ github.event.number }}" >> $GITHUB_ENV - echo "FLAMEGRAPH_NAME=nuclei (PR #${{ github.event.number }})" >> $GITHUB_ENV - - run: ./bin/nuclei -silent -update-templates - - run: ./bin/nuclei -silent -u "${TARGET_URL}" -profile-mem="${PROFILE_MEM}" - - uses: projectdiscovery/actions/flamegraph@v1 - id: flamegraph-cpu - with: - profile: "${{ env.PROFILE_MEM }}.cpu" - name: "${{ env.FLAMEGRAPH_NAME }} CPU profiles" - continue-on-error: true - - uses: projectdiscovery/actions/flamegraph@v1 - id: flamegraph-mem - with: - profile: "${{ env.PROFILE_MEM }}.mem" - name: "${{ env.FLAMEGRAPH_NAME }} memory profiles" - continue-on-error: true - - if: ${{ steps.flamegraph-mem.outputs.message == '' }} - run: | - echo "::notice::CPU flamegraph: ${{ steps.flamegraph-cpu.outputs.url }}" - echo "::notice::Memory (heap) flamegraph: ${{ steps.flamegraph-mem.outputs.url }}" + uses: ./.github/workflows/flamegraph.yaml perf-regression: + name: "Performance regression" needs: ["tests"] uses: ./.github/workflows/perf-regression.yaml diff --git a/.goreleaser.yml b/.goreleaser.yml index 2d2f61379..d95a631d4 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -11,8 +11,9 @@ builds: id: nuclei-cli env: - CGO_ENABLED=0 - goos: [windows,linux,darwin] - goarch: [amd64,'386',arm,arm64] + - GOEXPERIMENT=greenteagc + goos: [windows, linux, darwin] + goarch: [amd64, '386', arm, arm64] ignore: - goos: darwin goarch: '386'