ci: refactor workflows (#6728)

Signed-off-by: Dwi Siswanto <git@dw1.io>

.github/workflows/auto-merge.yaml

@@ -1,28 +1,24 @@
-name: 🤖 Auto Merge
+name: 🔀 Auto merge PR
 
 on:
-  pull_request_review:
+  # pull_request:
-    types: [submitted]
+  #   types: [opened, synchronize, reopened, ready_for_review]
+  # pull_request_review:
+  #   types: [submitted]
   workflow_run:
-    workflows: ["♾️ Compatibility Check"]
+    workflows: ["♾️ Compatibility Checks"]
-    types:
+    types: [completed]
-      - completed
 
 permissions:
+  contents: write
   pull-requests: write
-  issues: write
-  repository-projects: write
 
 jobs:
-  auto-merge:
+  auto-merge-dependabot:
     runs-on: ubuntu-latest
-    if: github.actor == 'dependabot[bot]'
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
     steps:
-      - uses: actions/checkout@v6
+      - uses: projectdiscovery/actions/pr/approve@v1
+      - uses: projectdiscovery/actions/pr/merge@v1
         with:
-          token: ${{ secrets.DEPENDABOT_PAT }}
+          auto: "true"
-
-      - uses: ahmadnassri/action-dependabot-auto-merge@v2
-        with:
-          github-token: ${{ secrets.DEPENDABOT_PAT }}
-          target: all

.github/workflows/compat-checks.yaml

@@ -7,8 +7,8 @@ on:
       - dev
 
 jobs:
-  check:
+  compat-checks:
-    if: github.actor == 'dependabot[bot]'
+    if: ${{ github.actor == 'dependabot[bot]' }}
     runs-on: ubuntu-latest
     permissions:
       contents: write

.github/workflows/flamegraph.yaml

@@ -0,0 +1,61 @@
+name: 📊 Flamegraph
+
+on:
+  workflow_call: {}
+
+jobs:
+  flamegraph:
+    name: "Flamegraph"
+    env:
+      PROFILE_MEM: "/tmp/nuclei-profile"
+      TARGET_URL: "http://honey.scanme.sh/-/?foo=bar"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: projectdiscovery/actions/setup/go@v1
+      - uses: projectdiscovery/actions/cache/go-rod-browser@v1
+      - uses: projectdiscovery/actions/cache/nuclei@v1
+      - run: make build
+
+      - name: "Setup environment (push)"
+        if: ${{ github.event_name == 'push' }}
+        run: |
+          echo "PROFILE_MEM=${PROFILE_MEM}-${GITHUB_REF_NAME}-${GITHUB_SHA}" >> $GITHUB_ENV
+          echo "FLAMEGRAPH_NAME=nuclei-${GITHUB_REF_NAME} (${GITHUB_SHA})" >> $GITHUB_ENV
+      - name: "Setup environment (pull_request)"
+        if: ${{ github.event_name == 'pull_request' }}
+        run: |
+          echo "PROFILE_MEM=${PROFILE_MEM}-pr-${{ github.event.number }}" >> $GITHUB_ENV
+          echo "FLAMEGRAPH_NAME=nuclei (PR #${{ github.event.number }})" >> $GITHUB_ENV
+
+      - run: ./bin/nuclei -update-templates
+      - run: |
+          ./bin/nuclei -silent -target="${TARGET_URL}" \
+            -disable-update-check \
+            -no-mhe -no-httpx \
+            -code -dast -file -headless \
+            -enable-self-contained -enable-global-matchers \
+            -rate-limit=0 \
+            -concurrency=250 \
+            -headless-concurrency=100 \
+            -payload-concurrency=250 \
+            -bulk-size=250 \
+            -headless-bulk-size=100 \
+            -profile-mem="${PROFILE_MEM}"
+
+      - uses: projectdiscovery/actions/flamegraph@v1
+        id: flamegraph-cpu
+        with:
+          profile: "${{ env.PROFILE_MEM }}.cpu"
+          name: "${{ env.FLAMEGRAPH_NAME }} CPU profiles"
+        continue-on-error: true
+      - uses: projectdiscovery/actions/flamegraph@v1
+        id: flamegraph-mem
+        with:
+          profile: "${{ env.PROFILE_MEM }}.mem"
+          name: "${{ env.FLAMEGRAPH_NAME }} memory profiles"
+        continue-on-error: true
+      - if: ${{ steps.flamegraph-mem.outputs.message == '' }}
+        run: |
+          echo "::notice::CPU flamegraph: ${{ steps.flamegraph-cpu.outputs.url }}"
+          echo "::notice::Memory (heap) flamegraph: ${{ steps.flamegraph-mem.outputs.url }}"

.github/workflows/generate-docs.yaml

@@ -4,11 +4,11 @@ on:
   push:
     branches:
       - dev
-  workflow_dispatch:
+  workflow_dispatch: {}
 
 jobs:
   publish-docs:
-    if: "${{ !endsWith(github.actor, '[bot]') }}"
+    if: ${{ !endsWith(github.actor, '[bot]') }}
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v6

.github/workflows/generate-pgo.yaml

@@ -1,56 +1,50 @@
 name: 👤 Generate PGO
 
 on:
-  push:
+  workflow_dispatch: {}
-    branches: ["dev"]
+  workflow_call: {}
-    paths:
-      - '**.go'
-      - '**.mod'
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-# TODO(dwisiswant0): https://go.dev/doc/pgo#merging-profiles
 
 jobs:
   pgo:
-    strategy:
-      matrix:
-        targets: [150]
     runs-on: ubuntu-latest
     if: github.repository == 'projectdiscovery/nuclei'
     permissions:
       contents: write
     env:
       PGO_FILE: "cmd/nuclei/default.pgo"
-      LIST_FILE: "/tmp/targets-${{ matrix.targets }}.txt"
+      TARGET: "https://honey.scanme.sh"
-      PROFILE_MEM: "/tmp/nuclei-profile-${{ matrix.targets }}-targets"
+      TARGET_COUNT: "100"
+      TARGET_LIST: "/tmp/targets.txt"
+      PROFILE_MEM: "/tmp/nuclei-profile"
     steps:
       - uses: actions/checkout@v6
-      - uses: projectdiscovery/actions/setup/git@v1
       - uses: projectdiscovery/actions/setup/go@v1
+      - uses: projectdiscovery/actions/cache/go-rod-browser@v1
       - uses: projectdiscovery/actions/cache/nuclei@v1
-      - name: Generate list
+      - run: |
-        run: for i in {1..${{ matrix.targets }}}; do echo "https://honey.scanme.sh/?_=${i}" >> "${LIST_FILE}"; done
+          for i in {1..${{ env.TARGET_COUNT }}}; do
-      # NOTE(dwisiswant0): use `-no-mhe` flag to get better samples.
+              echo "${{ env.TARGET }}/-/?_=${i}" >> "${{ env.TARGET_LIST }}";
-      - run: go run . -l "${LIST_FILE}" -profile-mem="${PROFILE_MEM}" -no-mhe
+          done
-        working-directory: cmd/nuclei/
+      - run: make build
-      - run: mv "${PROFILE_MEM}.cpu" ${PGO_FILE}
+      - run: ./bin/nuclei -update-templates
-      # NOTE(dwisiswant0): shall we prune $PGO_FILE git history?
+      - run: |
-      # if we prune it, this won't be linear since it requires a force-push.
+          ./bin/nuclei -silent -list="${TARGET_LIST}" \
-      # if we don't, the git objects will just keep growing bigger.
+            -disable-update-check \
-      # 
+            -no-mhe -no-httpx \
-      # Ref:
+            -code -dast -file -headless \
-      # - https://go.dev/blog/pgo#:~:text=We%20recommend%20committing%20default.pgo%20files%20to%20your%20repository
+            -enable-self-contained -enable-global-matchers \
-      # - https://gist.github.com/nottrobin/5758221
+            -rate-limit=0 \
-      - uses: projectdiscovery/actions/commit@v1
+            -concurrency=250 \
-        with:
+            -headless-concurrency=100 \
-          files: "${PGO_FILE}"
+            -payload-concurrency=250 \
-          message: "build: update PGO profile :robot:"
+            -bulk-size=250 \
-      - run: git push origin $GITHUB_REF
+            -headless-bulk-size=100 \
+            -profile-mem="${PROFILE_MEM}" \
+        env:
+          DISABLE_STDOUT: "1"
+      - run: mv "${PROFILE_MEM}.cpu" "${PGO_FILE}"
       - uses: actions/upload-artifact@v6
         with:
           name: "pgo"
           path: "${{ env.PGO_FILE }}"
+          overwrite: true

.github/workflows/govulncheck.yaml

@@ -3,7 +3,7 @@ name: 🐛 govulncheck
 on:
   schedule:
     - cron: '0 0 * * 0' # Weekly
-  workflow_dispatch:
+  workflow_dispatch: {}
 
 jobs:
   govulncheck:

.github/workflows/perf-regression.yaml

@@ -1,8 +1,8 @@
 name: 🔨 Performance Regression
 
 on:
-  workflow_call:
+  workflow_call: {}
-  workflow_dispatch:
+  workflow_dispatch: {}
 
 jobs:
   perf-regression:

.github/workflows/perf-test.yaml

@@ -1,43 +0,0 @@
-name: 🔨 Performance Test
-
-on:
-  schedule:
-    - cron: '0 0 * * 0' # Weekly
-  workflow_dispatch:
-
-jobs:
-  perf-test:
-    strategy:
-      matrix:
-        count: [50, 100, 150]
-    runs-on: ubuntu-latest
-    if: github.repository == 'projectdiscovery/nuclei'
-    env:
-      LIST_FILE: "/tmp/targets-${{ matrix.count }}.txt"
-      PROFILE_MEM: "/tmp/nuclei-perf-test-${{ matrix.count }}"
-    steps:
-      - uses: actions/checkout@v6
-      - uses: projectdiscovery/actions/setup/go@v1
-      - run: make verify
-      - name: Generate list
-        run: for i in {1..${{ matrix.count }}}; do echo "https://honey.scanme.sh/?_=${i}" >> "${LIST_FILE}"; done
-      - run: go run . -l "${LIST_FILE}" -profile-mem="${PROFILE_MEM}"
-        env:
-          NUCLEI_ARGS: host-error-stats
-        working-directory: cmd/nuclei/
-      - uses: projectdiscovery/actions/flamegraph@v1
-        id: flamegraph-cpu
-        with:
-          profile: "${{ env.PROFILE_MEM }}.cpu"
-          name: "${{ env.FLAMEGRAPH_NAME }} CPU profiles"
-        continue-on-error: true
-      - uses: projectdiscovery/actions/flamegraph@v1
-        id: flamegraph-mem
-        with:
-          profile: "${{ env.PROFILE_MEM }}.mem"
-          name: "${{ env.FLAMEGRAPH_NAME }} memory profiles"
-        continue-on-error: true
-      - if: ${{ steps.flamegraph-mem.outputs.message == '' }}
-        run: |
-          echo "::notice::CPU flamegraph: ${{ steps.flamegraph-cpu.outputs.url }}"
-          echo "::notice::Memory (heap) flamegraph: ${{ steps.flamegraph-mem.outputs.url }}"

.github/workflows/release.yaml

@@ -4,16 +4,28 @@ on:
   push:
     tags:
       - '*'
-  workflow_dispatch:
+  workflow_dispatch: {}
 
 jobs:
+  pgo:
+    name: "Generate PGO"
+    uses: ./.github/workflows/generate-pgo.yaml
+
   release:
+    name: "Release"
+    needs: ["pgo"]
     runs-on: ubuntu-latest-16-cores
     steps: 
       - uses: actions/checkout@v6
         with: 
           fetch-depth: 0
+      - uses: actions/download-artifact@v6
+        with:
+          name: "pgo"
+          path: "cmd/nuclei/"
       - uses: projectdiscovery/actions/setup/go@v1
+        with:
+          go-version: "stable"
       - uses: docker/login-action@v3 
         with:
           username: ${{ secrets.DOCKER_USERNAME }}

.github/workflows/tests.yaml

@@ -19,7 +19,7 @@ concurrency:
 jobs:
   lint:
     name: "Lint"
-    if: "${{ !endsWith(github.actor, '[bot]') }}"
+    if: ${{ !endsWith(github.actor, '[bot]') }}
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v6
@@ -157,42 +157,9 @@ jobs:
   flamegraph:
     name: "Flamegraph"
     needs: ["tests"]
-    env:
+    uses: ./.github/workflows/flamegraph.yaml
-      PROFILE_MEM: "/tmp/nuclei"
-      TARGET_URL: "http://scanme.sh/a/?b=c"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v6
-      - run: make build
-      - name: "Setup environment (push)"
-        if: ${{ github.event_name == 'push' }}
-        run: |
-          echo "PROFILE_MEM=${PROFILE_MEM}-${GITHUB_REF_NAME}-${GITHUB_SHA}" >> $GITHUB_ENV
-          echo "FLAMEGRAPH_NAME=nuclei-${GITHUB_REF_NAME} (${GITHUB_SHA})" >> $GITHUB_ENV
-      - name: "Setup environment (pull_request)"
-        if: ${{ github.event_name == 'pull_request' }}
-        run: |
-          echo "PROFILE_MEM=${PROFILE_MEM}-pr-${{ github.event.number }}" >> $GITHUB_ENV
-          echo "FLAMEGRAPH_NAME=nuclei (PR #${{ github.event.number }})" >> $GITHUB_ENV
-      - run: ./bin/nuclei -silent -update-templates
-      - run: ./bin/nuclei -silent -u "${TARGET_URL}" -profile-mem="${PROFILE_MEM}"
-      - uses: projectdiscovery/actions/flamegraph@v1
-        id: flamegraph-cpu
-        with:
-          profile: "${{ env.PROFILE_MEM }}.cpu"
-          name: "${{ env.FLAMEGRAPH_NAME }} CPU profiles"
-        continue-on-error: true
-      - uses: projectdiscovery/actions/flamegraph@v1
-        id: flamegraph-mem
-        with:
-          profile: "${{ env.PROFILE_MEM }}.mem"
-          name: "${{ env.FLAMEGRAPH_NAME }} memory profiles"
-        continue-on-error: true
-      - if: ${{ steps.flamegraph-mem.outputs.message == '' }}
-        run: |
-          echo "::notice::CPU flamegraph: ${{ steps.flamegraph-cpu.outputs.url }}"
-          echo "::notice::Memory (heap) flamegraph: ${{ steps.flamegraph-mem.outputs.url }}"
 
   perf-regression:
+    name: "Performance regression"
     needs: ["tests"]
     uses: ./.github/workflows/perf-regression.yaml

.goreleaser.yml

@@ -11,8 +11,9 @@ builds:
     id: nuclei-cli
     env:
       - CGO_ENABLED=0
-    goos: [windows,linux,darwin]
+      - GOEXPERIMENT=greenteagc
-    goarch: [amd64,'386',arm,arm64]
+    goos: [windows, linux, darwin]
+    goarch: [amd64, '386', arm, arm64]
     ignore:
       - goos: darwin
         goarch: '386'