Merge pull request #6712 from projectdiscovery/dwisiswant0/fix/trackers/add-gitlab-paginated-dup-issue-search

fix(trackers): add gitlab paginated dup issue search

cmd/nuclei/issue-tracker-config.yaml

@@ -55,6 +55,12 @@
 #  # these severity labels or tags (does not affect exporters. set those globally)
 #  deny-list:
 #    severity: low
+#  # duplicate-issue-check (optional) flag to enable duplicate tracking issue check
+#  duplicate-issue-check: false
+#  # duplicate-issue-page-size (optional) controls how many issues to fetch per page when searching for duplicates
+#  duplicate-issue-page-size: 100
+#  # duplicate-issue-max-pages (optional) limits how many pages to fetch when searching for duplicates (0 = no limit)
+#  duplicate-issue-max-pages: 0
 #
 # Gitea contains configuration options for a gitea issue tracker
 #gitea:

pkg/reporting/trackers/gitlab/gitlab.go

@@ -41,6 +41,12 @@ type Options struct {
 	DenyList *filters.Filter `yaml:"deny-list"`
 	// DuplicateIssueCheck is a bool to enable duplicate tracking issue check and update the newest
 	DuplicateIssueCheck bool `yaml:"duplicate-issue-check" default:"false"`
+	// DuplicateIssuePageSize controls how many issues are fetched per page when searching for duplicates.
+	// If unset or <=0, a default of 100 is used.
+	DuplicateIssuePageSize int `yaml:"duplicate-issue-page-size" default:"100"`
+	// DuplicateIssueMaxPages limits how many pages are fetched when searching for duplicates.
+	// If unset or <=0, all pages are fetched until exhaustion.
+	DuplicateIssueMaxPages int `yaml:"duplicate-issue-max-pages" default:"0"`
 
 	HttpClient *retryablehttp.Client `yaml:"-"`
 	OmitRaw    bool                  `yaml:"-"`
@@ -80,39 +86,36 @@ func (i *Integration) CreateIssue(event *output.ResultEvent) (*filters.CreateIss
 	}
 	customLabels := gitlab.LabelOptions(labels)
 	assigneeIDs := []int{i.userID}
+
+	var issue *gitlab.Issue
 	if i.options.DuplicateIssueCheck {
-		searchIn := "title"
-		searchState := "all"
-		issues, _, err := i.client.Issues.ListProjectIssues(i.options.ProjectName, &gitlab.ListProjectIssuesOptions{
-			In:     &searchIn,
-			State:  &searchState,
-			Search: &summary,
+		var err error
+		issue, err = i.findIssueByTitle(summary)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if issue != nil {
+		_, _, err := i.client.Notes.CreateIssueNote(i.options.ProjectName, issue.IID, &gitlab.CreateIssueNoteOptions{
+			Body: &description,
 		})
 		if err != nil {
 			return nil, err
 		}
-		if len(issues) > 0 {
-			issue := issues[0]
-			_, _, err := i.client.Notes.CreateIssueNote(i.options.ProjectName, issue.IID, &gitlab.CreateIssueNoteOptions{
-				Body: &description,
+		if issue.State == "closed" {
+			reopen := "reopen"
+			_, _, err := i.client.Issues.UpdateIssue(i.options.ProjectName, issue.IID, &gitlab.UpdateIssueOptions{
+				StateEvent: &reopen,
 			})
 			if err != nil {
 				return nil, err
 			}
-			if issue.State == "closed" {
-				reopen := "reopen"
-				_, _, err := i.client.Issues.UpdateIssue(i.options.ProjectName, issue.IID, &gitlab.UpdateIssueOptions{
-					StateEvent: &reopen,
-				})
-				if err != nil {
-					return nil, err
-				}
-			}
-			return &filters.CreateIssueResponse{
-				IssueID:  strconv.FormatInt(int64(issue.ID), 10),
-				IssueURL: issue.WebURL,
-			}, nil
 		}
+		return &filters.CreateIssueResponse{
+			IssueID:  strconv.FormatInt(int64(issue.ID), 10),
+			IssueURL: issue.WebURL,
+		}, nil
 	}
 	createdIssue, _, err := i.client.Issues.CreateIssue(i.options.ProjectName, &gitlab.CreateIssueOptions{
 		Title:       &summary,
@@ -134,23 +137,15 @@ func (i *Integration) Name() string {
 }
 
 func (i *Integration) CloseIssue(event *output.ResultEvent) error {
-	searchIn := "title"
-	searchState := "all"
-
 	summary := format.Summary(event)
-	issues, _, err := i.client.Issues.ListProjectIssues(i.options.ProjectName, &gitlab.ListProjectIssuesOptions{
-		In:     &searchIn,
-		State:  &searchState,
-		Search: &summary,
-	})
+	issue, err := i.findIssueByTitle(summary)
 	if err != nil {
 		return err
 	}
-	if len(issues) <= 0 {
+	if issue == nil {
 		return nil
 	}
 
-	issue := issues[0]
 	state := "close"
 	_, _, err = i.client.Issues.UpdateIssue(i.options.ProjectName, issue.IID, &gitlab.UpdateIssueOptions{
 		StateEvent: &state,
@@ -161,6 +156,49 @@ func (i *Integration) CloseIssue(event *output.ResultEvent) error {
 	return nil
 }
 
+func (i *Integration) findIssueByTitle(title string) (*gitlab.Issue, error) {
+	pageSize := i.options.DuplicateIssuePageSize
+	if pageSize <= 0 {
+		pageSize = 100
+	}
+	maxPages := i.options.DuplicateIssueMaxPages
+
+	searchIn := "title"
+	searchState := "all"
+	page := 1
+
+	for {
+		if maxPages > 0 && page > maxPages {
+			return nil, nil
+		}
+
+		issues, _, err := i.client.Issues.ListProjectIssues(i.options.ProjectName, &gitlab.ListProjectIssuesOptions{
+			In:     &searchIn,
+			State:  &searchState,
+			Search: &title,
+			ListOptions: gitlab.ListOptions{
+				Page:    page,
+				PerPage: pageSize,
+			},
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		for _, issue := range issues {
+			if issue.Title == title {
+				return issue, nil
+			}
+		}
+
+		if len(issues) < pageSize {
+			return nil, nil
+		}
+
+		page++
+	}
+}
+
 // ShouldFilter determines if an issue should be logged to this tracker
 func (i *Integration) ShouldFilter(event *output.ResultEvent) bool {
 	if i.options.AllowList != nil && !i.options.AllowList.GetMatch(event) {