diff --git a/cmd/nuclei/srv.yaml b/cmd/nuclei/srv.yaml deleted file mode 100644 index 198b397ce..000000000 --- a/cmd/nuclei/srv.yaml +++ /dev/null @@ -1,18 +0,0 @@ -id: basic-dns-a-example - -info: - name: Test DNS A Query Template - author: pdteam - severity: info - -dns: - - name: "{{FQDN}}" - type: SRV - class: inet - recursion: true - retries: 3 - matchers: - - type: word - part: all - words: - - "SRV" diff --git a/pkg/catalog/config/template.go b/pkg/catalog/config/template.go index 115bc814d..bbafefccf 100644 --- a/pkg/catalog/config/template.go +++ b/pkg/catalog/config/template.go @@ -154,11 +154,21 @@ func GetNucleiTemplatesIndex() (map[string]string, error) { if err == nil { for _, v := range records { if len(v) >= 2 { - index[v[0]] = v[1] + templateID := v[0] + templatePath := v[1] + // Normalize path for consistent comparison (handles Windows path issues) + normalizedPath := filepath.Clean(templatePath) + // Validate that the file actually exists (prevents stale entries from deleted files on Windows) + if fileutil.FileExists(normalizedPath) { + index[templateID] = normalizedPath + } } } + // Close file handle before returning + _ = f.Close() return index, nil } + _ = f.Close() } DefaultConfig.Logger.Error().Msgf("failed to read index file creating new one: %v", err) } @@ -177,13 +187,15 @@ func GetNucleiTemplatesIndex() (map[string]string, error) { if d.IsDir() || !IsTemplateWithRoot(path, DefaultConfig.TemplatesDirectory) || stringsutil.ContainsAny(path, ignoreDirs...) { return nil } + // Normalize path for consistent comparison (handles Windows path issues) + normalizedPath := filepath.Clean(path) // get template id from file - id, err := getTemplateID(path) + id, err := getTemplateID(normalizedPath) if err != nil || id == "" { - DefaultConfig.Logger.Verbose().Msgf("failed to get template id from file=%v got id=%v err=%v", path, id, err) + DefaultConfig.Logger.Verbose().Msgf("failed to get template id from file=%v got id=%v err=%v", normalizedPath, id, err) return nil } - index[id] = path + index[id] = normalizedPath return nil }) return index, err diff --git a/pkg/installer/template.go b/pkg/installer/template.go index 577f8c252..00a9982da 100644 --- a/pkg/installer/template.go +++ b/pkg/installer/template.go @@ -19,6 +19,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/external/customtemplates" "github.com/projectdiscovery/utils/errkit" fileutil "github.com/projectdiscovery/utils/file" + mapsutil "github.com/projectdiscovery/utils/maps" stringsutil "github.com/projectdiscovery/utils/strings" updateutils "github.com/projectdiscovery/utils/update" ) @@ -134,7 +135,8 @@ func (t *TemplateManager) installTemplatesAt(dir string) error { } // write templates to disk - if err := t.writeTemplatesToDisk(ghrd, dir); err != nil { + _, err = t.writeTemplatesToDisk(ghrd, dir) + if err != nil { return errkit.Wrapf(err, "failed to write templates to disk at %s", dir) } gologger.Info().Msgf("Successfully installed nuclei-templates at %s", dir) @@ -169,10 +171,26 @@ func (t *TemplateManager) updateTemplatesAt(dir string) error { } // write templates to disk - if err := t.writeTemplatesToDisk(ghrd, dir); err != nil { + writtenPaths, err := t.writeTemplatesToDisk(ghrd, dir) + if err != nil { return err } + // cleanup orphaned templates that exist locally but weren't in the new release + if err := t.cleanupOrphanedTemplates(dir, writtenPaths); err != nil { + // log warning but don't fail the update + gologger.Warning().Msgf("failed to cleanup orphaned templates: %s", err) + } else { + // Regenerate metadata (index and checksum) after successful cleanup to ensure + // metadata accurately reflects the cleaned template tree. This prevents stale + // index entries and checksum entries for deleted templates. + if err := t.regenerateTemplateMetadata(dir); err != nil { + // Log warning but don't fail the update - metadata will be out of sync + // but templates are cleaned up correctly + gologger.Warning().Msgf("failed to regenerate template metadata after cleanup: %s", err) + } + } + // get checksums from new templates newchecksums, err := t.getChecksumFromDir(dir) if err != nil { @@ -279,8 +297,9 @@ func (t *TemplateManager) getAbsoluteFilePath(templateDir, uri string, f fs.File return newPath } -// writeChecksumFileInDir is actual method responsible for writing all templates to directory -func (t *TemplateManager) writeTemplatesToDisk(ghrd *updateutils.GHReleaseDownloader, dir string) error { +// writeTemplatesToDisk writes all templates to disk and returns a map of written file paths +// The returned map contains absolute paths of all template files that were successfully written +func (t *TemplateManager) writeTemplatesToDisk(ghrd *updateutils.GHReleaseDownloader, dir string) (*mapsutil.SyncLockMap[string, struct{}], error) { localTemplatesIndex, err := config.GetNucleiTemplatesIndex() if err != nil { gologger.Warning().Msgf("failed to get local nuclei-templates index: %s", err) @@ -289,6 +308,9 @@ func (t *TemplateManager) writeTemplatesToDisk(ghrd *updateutils.GHReleaseDownlo } } + // Track all paths that are successfully written during this update + writtenPaths := mapsutil.NewSyncLockMap[string, struct{}]() + callbackFunc := func(uri string, f fs.FileInfo, r io.Reader) error { writePath := t.getAbsoluteFilePath(dir, uri, f) if writePath == "" { @@ -312,6 +334,8 @@ func (t *TemplateManager) writeTemplatesToDisk(ghrd *updateutils.GHReleaseDownlo if err := os.WriteFile(writePath, bin, f.Mode()); err != nil { return errkit.Wrapf(err, "failed to write file %s", uri) } + // Track the new path as written + _ = writtenPaths.Set(writePath, struct{}{}) // after successful write, remove old template if err := os.Remove(oldPath); err != nil { gologger.Warning().Msgf("failed to remove old template %s: %s", oldPath, err) @@ -321,24 +345,29 @@ func (t *TemplateManager) writeTemplatesToDisk(ghrd *updateutils.GHReleaseDownlo } } // no change in template Path of official templates - return os.WriteFile(writePath, bin, f.Mode()) + if err := os.WriteFile(writePath, bin, f.Mode()); err != nil { + return errkit.Wrapf(err, "failed to write file %s", uri) + } + // Track successfully written paths + _ = writtenPaths.Set(writePath, struct{}{}) + return nil } err = ghrd.DownloadSourceWithCallback(!HideProgressBar, callbackFunc) if err != nil { - return errkit.Wrap(err, "failed to download templates") + return nil, errkit.Wrap(err, "failed to download templates") } if err := config.DefaultConfig.WriteTemplatesConfig(); err != nil { - return errkit.Wrap(err, "failed to write templates config") + return nil, errkit.Wrap(err, "failed to write templates config") } // update ignore hash after writing new templates if err := config.DefaultConfig.UpdateNucleiIgnoreHash(); err != nil { - return errkit.Wrap(err, "failed to update nuclei ignore hash") + return nil, errkit.Wrap(err, "failed to update nuclei ignore hash") } // update templates version in config file if err := config.DefaultConfig.SetTemplatesVersion(ghrd.Latest.GetTagName()); err != nil { - return errkit.Wrap(err, "failed to update templates version") + return nil, errkit.Wrap(err, "failed to update templates version") } PurgeEmptyDirectories(dir) @@ -348,11 +377,11 @@ func (t *TemplateManager) writeTemplatesToDisk(ghrd *updateutils.GHReleaseDownlo index, err := config.GetNucleiTemplatesIndex() if err != nil { - return errkit.Wrap(err, "failed to get nuclei templates index") + return nil, errkit.Wrap(err, "failed to get nuclei templates index") } if err = config.DefaultConfig.WriteTemplatesIndex(index); err != nil { - return errkit.Wrap(err, "failed to write nuclei templates index") + return nil, errkit.Wrap(err, "failed to write nuclei templates index") } if !HideReleaseNotes { @@ -371,7 +400,161 @@ func (t *TemplateManager) writeTemplatesToDisk(ghrd *updateutils.GHReleaseDownlo } // after installation, create and write checksums to .checksum file - return t.writeChecksumFileInDir(dir) + if err := t.writeChecksumFileInDir(dir); err != nil { + return nil, err + } + + return writtenPaths, nil +} + +// cleanupOrphanedTemplates removes template files that exist locally but were not part of the new release +// It scans the templates directory for template files and deletes those that are not in the writtenPaths set +// This function handles empty directories gracefully - if the directory is empty, no orphaned files will be found +func (t *TemplateManager) cleanupOrphanedTemplates(dir string, writtenPaths *mapsutil.SyncLockMap[string, struct{}]) error { + absDir, err := filepath.Abs(dir) + if err != nil { + return errkit.Wrapf(err, "failed to get absolute path of templates directory") + } + // Use Clean to normalize the path consistently (handles Windows paths better) + absDir = filepath.Clean(absDir) + + // If directory doesn't exist, there's nothing to clean up + if !fileutil.FolderExists(absDir) { + return nil + } + + // Normalize all written paths to absolute paths for comparison + normalizedWrittenPaths := mapsutil.NewSyncLockMap[string, struct{}]() + for path := range writtenPaths.GetAll() { + absPath, err := filepath.Abs(path) + if err == nil { + // Use Clean to normalize the path consistently (handles Windows paths better) + absPath = filepath.Clean(absPath) + _ = normalizedWrittenPaths.Set(absPath, struct{}{}) + } + } + + // Get custom template directories to exclude + customDirs := config.DefaultConfig.GetAllCustomTemplateDirs() + customDirAbs := make([]string, 0, len(customDirs)) + for _, customDir := range customDirs { + if absCustomDir, err := filepath.Abs(customDir); err == nil { + // Use Clean to normalize the path consistently (handles Windows paths better) + absCustomDir = filepath.Clean(absCustomDir) + customDirAbs = append(customDirAbs, absCustomDir) + } + } + + var orphanedFiles []string + + // Walk the templates directory to find all template files + err = filepath.WalkDir(absDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + // Log but continue walking + gologger.Debug().Msgf("error accessing path %s during orphan cleanup: %s", path, err) + return nil + } + + // Skip directories + if d.IsDir() { + return nil + } + + absPath, err := filepath.Abs(path) + if err != nil { + return nil + } + // Use Clean to normalize the path consistently (handles Windows paths better) + absPath = filepath.Clean(absPath) + + // Skip custom template directories + for _, customDir := range customDirAbs { + if strings.HasPrefix(absPath, customDir) { + return nil + } + } + + // Only process template files + if !config.IsTemplate(absPath) { + return nil + } + + // Skip if this file was written in the new release + if normalizedWrittenPaths.Has(absPath) { + return nil + } + + // This is an orphaned template file + orphanedFiles = append(orphanedFiles, absPath) + return nil + }) + + if err != nil { + return errkit.Wrapf(err, "failed to walk templates directory for orphan cleanup") + } + + // Delete orphaned files + for _, orphanPath := range orphanedFiles { + if err := os.Remove(orphanPath); err != nil { + if !os.IsNotExist(err) { + gologger.Warning().Msgf("failed to remove orphaned template %s: %s", orphanPath, err) + } + } else { + gologger.Debug().Msgf("removed orphaned template: %s", orphanPath) + } + } + + if len(orphanedFiles) > 0 { + gologger.Info().Msgf("cleaned up %d orphaned template file(s)", len(orphanedFiles)) + } + + return nil +} + +// regenerateTemplateMetadata regenerates template index and checksum files after cleanup operations. +// This ensures the metadata accurately reflects the current state of template files on disk. +func (t *TemplateManager) regenerateTemplateMetadata(dir string) error { + // Purge empty directories that may have been left after cleanup + PurgeEmptyDirectories(dir) + + // Ensure templates directory exists (it may have been purged if empty) + if !fileutil.FolderExists(dir) { + if err := os.MkdirAll(dir, 0755); err != nil { + return errkit.Wrapf(err, "failed to recreate templates directory %s after purge", dir) + } + } + + // Remove old index file and regenerate it from current templates on disk + indexFilePath := config.DefaultConfig.GetTemplateIndexFilePath() + if err := os.Remove(indexFilePath); err != nil && !os.IsNotExist(err) { + return errkit.Wrapf(err, "failed to remove old index file %s", indexFilePath) + } + + // Force regeneration by ensuring the file doesn't exist (handles Windows file handle issues) + // GetNucleiTemplatesIndex will scan the directory if the file doesn't exist + index, err := config.GetNucleiTemplatesIndex() + if err != nil { + return errkit.Wrap(err, "failed to regenerate nuclei templates index after cleanup") + } + + // Filter out any entries that don't actually exist on disk (Windows file deletion timing issues) + filteredIndex := make(map[string]string) + for id, path := range index { + if fileutil.FileExists(path) { + filteredIndex[id] = path + } + } + + if err = config.DefaultConfig.WriteTemplatesIndex(filteredIndex); err != nil { + return errkit.Wrap(err, "failed to write nuclei templates index after cleanup") + } + + // Regenerate checksum file to reflect current templates on disk + if err := t.writeChecksumFileInDir(dir); err != nil { + return errkit.Wrap(err, "failed to regenerate checksum file after cleanup") + } + + return nil } // getChecksumFromDir returns a map containing checksums (md5 hash) of all yaml files (with .yaml extension) diff --git a/pkg/installer/template_test.go b/pkg/installer/template_test.go index 435797b1f..57ca67901 100644 --- a/pkg/installer/template_test.go +++ b/pkg/installer/template_test.go @@ -7,6 +7,8 @@ import ( "testing" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" + fileutil "github.com/projectdiscovery/utils/file" + mapsutil "github.com/projectdiscovery/utils/maps" "github.com/stretchr/testify/require" ) @@ -98,3 +100,649 @@ func TestIsOutdatedVersion(t *testing.T) { }) } } + +func TestCleanupOrphanedTemplates(t *testing.T) { + HideProgressBar = true + + tm := &TemplateManager{} + + t.Run("removes orphaned templates", func(t *testing.T) { + // Create temporary directories + tmpDir, err := os.MkdirTemp("", "nuclei-cleanup-test-*") + require.NoError(t, err) + defer func() { + _ = os.RemoveAll(tmpDir) + }() + + cfgdir, err := os.MkdirTemp("", "nuclei-config-*") + require.NoError(t, err) + defer func() { + _ = os.RemoveAll(cfgdir) + }() + + config.DefaultConfig.SetConfigDir(cfgdir) + config.DefaultConfig.SetTemplatesDir(tmpDir) + + // Create subdirectories for templates + templatesDir1 := filepath.Join(tmpDir, "cves", "2023") + templatesDir2 := filepath.Join(tmpDir, "exposures", "configs") + require.NoError(t, os.MkdirAll(templatesDir1, 0755)) + require.NoError(t, os.MkdirAll(templatesDir2, 0755)) + + // Create template files + template1 := filepath.Join(templatesDir1, "CVE-2023-1234.yaml") + template2 := filepath.Join(templatesDir1, "CVE-2023-5678.yaml") + template3 := filepath.Join(templatesDir2, "git-config-exposure.yaml") + orphanedTemplate1 := filepath.Join(templatesDir1, "old-template.yaml") + orphanedTemplate2 := filepath.Join(templatesDir2, "removed-template.yaml") + + // Write valid template files + templateContent := `id: test-template +info: + name: Test Template + author: test + severity: info` + require.NoError(t, os.WriteFile(template1, []byte(templateContent), 0644)) + require.NoError(t, os.WriteFile(template2, []byte(templateContent), 0644)) + require.NoError(t, os.WriteFile(template3, []byte(templateContent), 0644)) + require.NoError(t, os.WriteFile(orphanedTemplate1, []byte(templateContent), 0644)) + require.NoError(t, os.WriteFile(orphanedTemplate2, []byte(templateContent), 0644)) + + // Simulate written paths from new release (only template1, template2, template3) + writtenPaths := mapsutil.NewSyncLockMap[string, struct{}]() + absTemplate1, _ := filepath.Abs(template1) + absTemplate2, _ := filepath.Abs(template2) + absTemplate3, _ := filepath.Abs(template3) + // Normalize paths consistently (same as cleanupOrphanedTemplates does) + absTemplate1 = filepath.Clean(absTemplate1) + absTemplate2 = filepath.Clean(absTemplate2) + absTemplate3 = filepath.Clean(absTemplate3) + _ = writtenPaths.Set(absTemplate1, struct{}{}) + _ = writtenPaths.Set(absTemplate2, struct{}{}) + _ = writtenPaths.Set(absTemplate3, struct{}{}) + + // Run cleanup + err = tm.cleanupOrphanedTemplates(tmpDir, writtenPaths) + require.NoError(t, err) + + // Verify orphaned templates were removed + require.NoFileExists(t, orphanedTemplate1, "orphaned template should be removed") + require.NoFileExists(t, orphanedTemplate2, "orphaned template should be removed") + + // Verify non-orphaned templates still exist + require.FileExists(t, template1, "template from new release should exist") + require.FileExists(t, template2, "template from new release should exist") + require.FileExists(t, template3, "template from new release should exist") + }) + + t.Run("preserves custom templates", func(t *testing.T) { + // Create temporary directories + tmpDir, err := os.MkdirTemp("", "nuclei-cleanup-custom-test-*") + require.NoError(t, err) + defer func() { + _ = os.RemoveAll(tmpDir) + }() + + cfgdir, err := os.MkdirTemp("", "nuclei-config-*") + require.NoError(t, err) + defer func() { + _ = os.RemoveAll(cfgdir) + }() + + config.DefaultConfig.SetConfigDir(cfgdir) + config.DefaultConfig.SetTemplatesDir(tmpDir) + + // Create custom template directory + customGitHubDir := filepath.Join(tmpDir, "github", "owner", "repo") + require.NoError(t, os.MkdirAll(customGitHubDir, 0755)) + + // Create custom template file + customTemplate := filepath.Join(customGitHubDir, "custom-template.yaml") + templateContent := `id: custom-template +info: + name: Custom Template + author: test + severity: info` + require.NoError(t, os.WriteFile(customTemplate, []byte(templateContent), 0644)) + + // Empty written paths (simulating no custom templates in new release) + writtenPaths := mapsutil.NewSyncLockMap[string, struct{}]() + + // Run cleanup + err = tm.cleanupOrphanedTemplates(tmpDir, writtenPaths) + require.NoError(t, err) + + // Verify custom template was NOT removed + require.FileExists(t, customTemplate, "custom template should be preserved") + }) + + t.Run("skips non-template files", func(t *testing.T) { + // Create temporary directories + tmpDir, err := os.MkdirTemp("", "nuclei-cleanup-nontemplate-test-*") + require.NoError(t, err) + defer func() { + _ = os.RemoveAll(tmpDir) + }() + + cfgdir, err := os.MkdirTemp("", "nuclei-config-*") + require.NoError(t, err) + defer func() { + _ = os.RemoveAll(cfgdir) + }() + + config.DefaultConfig.SetConfigDir(cfgdir) + config.DefaultConfig.SetTemplatesDir(tmpDir) + + // Create non-template files + readmeFile := filepath.Join(tmpDir, "README.md") + configFile := filepath.Join(tmpDir, "cves.json") + checksumFile := filepath.Join(tmpDir, ".checksum") + + require.NoError(t, os.WriteFile(readmeFile, []byte("# Templates"), 0644)) + require.NoError(t, os.WriteFile(configFile, []byte("{}"), 0644)) + require.NoError(t, os.WriteFile(checksumFile, []byte(""), 0644)) + + // Empty written paths + writtenPaths := mapsutil.NewSyncLockMap[string, struct{}]() + + // Run cleanup + err = tm.cleanupOrphanedTemplates(tmpDir, writtenPaths) + require.NoError(t, err) + + // Verify non-template files were NOT removed + require.FileExists(t, readmeFile, "README.md should be preserved") + require.FileExists(t, configFile, "config file should be preserved") + require.FileExists(t, checksumFile, "checksum file should be preserved") + }) + + t.Run("handles empty written paths", func(t *testing.T) { + // Create temporary directories + tmpDir, err := os.MkdirTemp("", "nuclei-cleanup-empty-test-*") + require.NoError(t, err) + defer func() { + _ = os.RemoveAll(tmpDir) + }() + + cfgdir, err := os.MkdirTemp("", "nuclei-config-*") + require.NoError(t, err) + defer func() { + _ = os.RemoveAll(cfgdir) + }() + + config.DefaultConfig.SetConfigDir(cfgdir) + config.DefaultConfig.SetTemplatesDir(tmpDir) + + // Create template files + template1 := filepath.Join(tmpDir, "template1.yaml") + templateContent := `id: test-template +info: + name: Test Template + author: test + severity: info` + require.NoError(t, os.WriteFile(template1, []byte(templateContent), 0644)) + + // Empty written paths + writtenPaths := mapsutil.NewSyncLockMap[string, struct{}]() + + // Run cleanup + err = tm.cleanupOrphanedTemplates(tmpDir, writtenPaths) + require.NoError(t, err) + + // Verify template was removed (since it's not in written paths) + require.NoFileExists(t, template1, "template should be removed when not in written paths") + }) + + t.Run("handles relative and absolute paths correctly", func(t *testing.T) { + // Create temporary directories + tmpDir, err := os.MkdirTemp("", "nuclei-cleanup-path-test-*") + require.NoError(t, err) + defer func() { + _ = os.RemoveAll(tmpDir) + }() + + cfgdir, err := os.MkdirTemp("", "nuclei-config-*") + require.NoError(t, err) + defer func() { + _ = os.RemoveAll(cfgdir) + }() + + config.DefaultConfig.SetConfigDir(cfgdir) + config.DefaultConfig.SetTemplatesDir(tmpDir) + + // Create template file + template1 := filepath.Join(tmpDir, "template1.yaml") + templateContent := `id: test-template +info: + name: Test Template + author: test + severity: info` + require.NoError(t, os.WriteFile(template1, []byte(templateContent), 0644)) + + // Use relative path in written paths + writtenPaths := mapsutil.NewSyncLockMap[string, struct{}]() + _ = writtenPaths.Set(template1, struct{}{}) // relative path + + // Run cleanup - should normalize paths correctly + err = tm.cleanupOrphanedTemplates(tmpDir, writtenPaths) + require.NoError(t, err) + + // Verify template was NOT removed (it was in written paths) + require.FileExists(t, template1, "template should be preserved when in written paths") + }) + + t.Run("handles empty templates directory", func(t *testing.T) { + // Create temporary directories + tmpDir, err := os.MkdirTemp("", "nuclei-cleanup-empty-dir-test-*") + require.NoError(t, err) + defer func() { + _ = os.RemoveAll(tmpDir) + }() + + cfgdir, err := os.MkdirTemp("", "nuclei-config-*") + require.NoError(t, err) + defer func() { + _ = os.RemoveAll(cfgdir) + }() + + config.DefaultConfig.SetConfigDir(cfgdir) + config.DefaultConfig.SetTemplatesDir(tmpDir) + + // Directory exists but is empty (user deleted all templates) + require.True(t, fileutil.FolderExists(tmpDir), "templates directory should exist") + + // Written paths from new release + writtenPaths := mapsutil.NewSyncLockMap[string, struct{}]() + + // Run cleanup - should handle empty directory gracefully + err = tm.cleanupOrphanedTemplates(tmpDir, writtenPaths) + require.NoError(t, err, "cleanup should handle empty directory without error") + + // Directory should still exist after cleanup + require.True(t, fileutil.FolderExists(tmpDir), "templates directory should still exist") + }) + + t.Run("handles non-existent directory gracefully", func(t *testing.T) { + // Use a non-existent directory path + nonExistentDir := "/tmp/nuclei-test-non-existent-dir-12345" + + // Ensure it doesn't exist + _ = os.RemoveAll(nonExistentDir) + require.False(t, fileutil.FolderExists(nonExistentDir), "directory should not exist") + + writtenPaths := mapsutil.NewSyncLockMap[string, struct{}]() + + // Run cleanup - should handle non-existent directory gracefully + err := tm.cleanupOrphanedTemplates(nonExistentDir, writtenPaths) + require.NoError(t, err, "cleanup should handle non-existent directory without error") + }) +} + +func TestRegenerateTemplateMetadata(t *testing.T) { + HideProgressBar = true + tm := &TemplateManager{} + + t.Run("creates index and checksum files", func(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "nuclei-metadata-test-*") + require.NoError(t, err) + defer func() { + _ = os.RemoveAll(tmpDir) + }() + + cfgdir, err := os.MkdirTemp("", "nuclei-config-*") + require.NoError(t, err) + defer func() { + _ = os.RemoveAll(cfgdir) + }() + + config.DefaultConfig.SetConfigDir(cfgdir) + config.DefaultConfig.SetTemplatesDir(tmpDir) + + // Create template files with unique IDs + template1 := filepath.Join(tmpDir, "template1.yaml") + template2 := filepath.Join(tmpDir, "cves", "template2.yaml") + require.NoError(t, os.MkdirAll(filepath.Dir(template2), 0755)) + + template1Content := `id: template-one +info: + name: Template One + author: test + severity: info` + template2Content := `id: template-two +info: + name: Template Two + author: test + severity: high` + + require.NoError(t, os.WriteFile(template1, []byte(template1Content), 0644)) + require.NoError(t, os.WriteFile(template2, []byte(template2Content), 0644)) + + // Regenerate metadata + err = tm.regenerateTemplateMetadata(tmpDir) + require.NoError(t, err) + + // Verify index file was created + indexPath := config.DefaultConfig.GetTemplateIndexFilePath() + require.FileExists(t, indexPath, "template index file should be created") + + // Verify checksum file was created + checksumPath := config.DefaultConfig.GetChecksumFilePath() + require.FileExists(t, checksumPath, "checksum file should be created") + + // Verify index contains both templates + index, err := config.GetNucleiTemplatesIndex() + require.NoError(t, err) + require.Contains(t, index, "template-one", "index should contain template-one") + require.Contains(t, index, "template-two", "index should contain template-two") + + // Verify checksum file contains both templates + checksums, err := tm.getChecksumFromDir(tmpDir) + require.NoError(t, err) + require.Contains(t, checksums, template1, "checksum should contain template1") + require.Contains(t, checksums, template2, "checksum should contain template2") + }) + + t.Run("excludes deleted templates from index after cleanup", func(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "nuclei-metadata-cleanup-test-*") + require.NoError(t, err) + defer func() { + _ = os.RemoveAll(tmpDir) + }() + + cfgdir, err := os.MkdirTemp("", "nuclei-config-*") + require.NoError(t, err) + defer func() { + _ = os.RemoveAll(cfgdir) + }() + + config.DefaultConfig.SetConfigDir(cfgdir) + config.DefaultConfig.SetTemplatesDir(tmpDir) + + // Create template files + template1 := filepath.Join(tmpDir, "kept-template.yaml") + template2 := filepath.Join(tmpDir, "deleted-template.yaml") + orphanedTemplate := filepath.Join(tmpDir, "orphaned-template.yaml") + + template1Content := `id: test-template-1 +info: + name: Test Template 1 + author: test + severity: info` + template2Content := `id: test-template-2 +info: + name: Test Template 2 + author: test + severity: info` + orphanedContent := `id: test-template-orphaned +info: + name: Test Template Orphaned + author: test + severity: info` + + require.NoError(t, os.WriteFile(template1, []byte(template1Content), 0644)) + require.NoError(t, os.WriteFile(template2, []byte(template2Content), 0644)) + require.NoError(t, os.WriteFile(orphanedTemplate, []byte(orphanedContent), 0644)) + + // Create initial index with all templates (simulating state before cleanup) + initialIndex := map[string]string{ + "test-template-1": template1, + "test-template-2": template2, + "test-template-orphaned": orphanedTemplate, + } + err = config.DefaultConfig.WriteTemplatesIndex(initialIndex) + require.NoError(t, err) + + // Verify initial index contains all templates + index, err := config.GetNucleiTemplatesIndex() + require.NoError(t, err) + require.Contains(t, index, "test-template-orphaned", "initial index should contain orphaned template") + + // Simulate cleanup: remove orphaned template + writtenPaths := mapsutil.NewSyncLockMap[string, struct{}]() + absTemplate1, _ := filepath.Abs(template1) + // Normalize path consistently (same as cleanupOrphanedTemplates does) + absTemplate1 = filepath.Clean(absTemplate1) + _ = writtenPaths.Set(absTemplate1, struct{}{}) + + err = tm.cleanupOrphanedTemplates(tmpDir, writtenPaths) + require.NoError(t, err) + require.NoFileExists(t, orphanedTemplate, "orphaned template should be deleted") + require.NoFileExists(t, template2, "template2 should be deleted since it's not in writtenPaths") + + // Regenerate metadata after cleanup + err = tm.regenerateTemplateMetadata(tmpDir) + require.NoError(t, err) + + // Verify index no longer contains deleted template + index, err = config.GetNucleiTemplatesIndex() + require.NoError(t, err) + require.NotContains(t, index, "test-template-orphaned", "index should not contain deleted orphaned template") + require.Contains(t, index, "test-template-1", "index should still contain kept template") + require.NotContains(t, index, "test-template-2", "index should not contain template that was deleted but not cleaned") + }) + + t.Run("excludes deleted templates from checksum after cleanup", func(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "nuclei-checksum-cleanup-test-*") + require.NoError(t, err) + defer func() { + _ = os.RemoveAll(tmpDir) + }() + + cfgdir, err := os.MkdirTemp("", "nuclei-config-*") + require.NoError(t, err) + defer func() { + _ = os.RemoveAll(cfgdir) + }() + + config.DefaultConfig.SetConfigDir(cfgdir) + config.DefaultConfig.SetTemplatesDir(tmpDir) + + // Create template files + keptTemplate := filepath.Join(tmpDir, "kept.yaml") + orphanedTemplate := filepath.Join(tmpDir, "orphaned.yaml") + + templateContent := `id: test-template +info: + name: Test Template + author: test + severity: info` + + require.NoError(t, os.WriteFile(keptTemplate, []byte(templateContent), 0644)) + require.NoError(t, os.WriteFile(orphanedTemplate, []byte(templateContent), 0644)) + + // Create initial checksum with both templates + err = tm.writeChecksumFileInDir(tmpDir) + require.NoError(t, err) + + // Verify initial checksum contains both templates + initialChecksums, err := tm.getChecksumFromDir(tmpDir) + require.NoError(t, err) + require.Contains(t, initialChecksums, orphanedTemplate, "initial checksum should contain orphaned template") + + // Simulate cleanup: remove orphaned template + writtenPaths := mapsutil.NewSyncLockMap[string, struct{}]() + absKept, _ := filepath.Abs(keptTemplate) + // Normalize path consistently (same as cleanupOrphanedTemplates does) + absKept = filepath.Clean(absKept) + _ = writtenPaths.Set(absKept, struct{}{}) + + err = tm.cleanupOrphanedTemplates(tmpDir, writtenPaths) + require.NoError(t, err) + require.NoFileExists(t, orphanedTemplate, "orphaned template should be deleted") + + // Regenerate metadata after cleanup + err = tm.regenerateTemplateMetadata(tmpDir) + require.NoError(t, err) + + // Verify checksum no longer contains deleted template + checksums, err := tm.getChecksumFromDir(tmpDir) + require.NoError(t, err) + require.NotContains(t, checksums, orphanedTemplate, "checksum should not contain deleted orphaned template") + require.Contains(t, checksums, keptTemplate, "checksum should still contain kept template") + }) + + t.Run("cleanup and metadata regeneration integration", func(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "nuclei-integration-test-*") + require.NoError(t, err) + defer func() { + _ = os.RemoveAll(tmpDir) + }() + + cfgdir, err := os.MkdirTemp("", "nuclei-config-*") + require.NoError(t, err) + defer func() { + _ = os.RemoveAll(cfgdir) + }() + + config.DefaultConfig.SetConfigDir(cfgdir) + config.DefaultConfig.SetTemplatesDir(tmpDir) + + // Create multiple templates + template1 := filepath.Join(tmpDir, "cves", "2023", "cve1.yaml") + template2 := filepath.Join(tmpDir, "cves", "2023", "cve2.yaml") + orphaned1 := filepath.Join(tmpDir, "cves", "2022", "old-cve.yaml") + orphaned2 := filepath.Join(tmpDir, "exposures", "old-exposure.yaml") + + require.NoError(t, os.MkdirAll(filepath.Dir(template1), 0755)) + require.NoError(t, os.MkdirAll(filepath.Dir(orphaned1), 0755)) + require.NoError(t, os.MkdirAll(filepath.Dir(orphaned2), 0755)) + + template1Content := `id: cve1 +info: + name: CVE1 + author: test + severity: info` + template2Content := `id: cve2 +info: + name: CVE2 + author: test + severity: info` + orphaned1Content := `id: old-cve +info: + name: Old CVE + author: test + severity: info` + orphaned2Content := `id: old-exposure +info: + name: Old Exposure + author: test + severity: info` + + require.NoError(t, os.WriteFile(template1, []byte(template1Content), 0644)) + require.NoError(t, os.WriteFile(template2, []byte(template2Content), 0644)) + require.NoError(t, os.WriteFile(orphaned1, []byte(orphaned1Content), 0644)) + require.NoError(t, os.WriteFile(orphaned2, []byte(orphaned2Content), 0644)) + + // Simulate written paths from new release + writtenPaths := mapsutil.NewSyncLockMap[string, struct{}]() + absTemplate1, _ := filepath.Abs(template1) + absTemplate2, _ := filepath.Abs(template2) + // Normalize paths consistently (same as cleanupOrphanedTemplates does) + absTemplate1 = filepath.Clean(absTemplate1) + absTemplate2 = filepath.Clean(absTemplate2) + _ = writtenPaths.Set(absTemplate1, struct{}{}) + _ = writtenPaths.Set(absTemplate2, struct{}{}) + + // Perform cleanup + err = tm.cleanupOrphanedTemplates(tmpDir, writtenPaths) + require.NoError(t, err) + require.NoFileExists(t, orphaned1, "orphaned template 1 should be deleted") + require.NoFileExists(t, orphaned2, "orphaned template 2 should be deleted") + + // Regenerate metadata (simulating what updateTemplatesAt does) + err = tm.regenerateTemplateMetadata(tmpDir) + require.NoError(t, err) + + // Verify index only contains kept templates + index, err := config.GetNucleiTemplatesIndex() + require.NoError(t, err) + require.Contains(t, index, "cve1", "index should contain kept template cve1") + require.Contains(t, index, "cve2", "index should contain kept template cve2") + require.NotContains(t, index, "old-cve", "index should not contain deleted template") + require.NotContains(t, index, "old-exposure", "index should not contain deleted template") + + // Verify checksum only contains kept templates + checksums, err := tm.getChecksumFromDir(tmpDir) + require.NoError(t, err) + require.Contains(t, checksums, template1, "checksum should contain kept template1") + require.Contains(t, checksums, template2, "checksum should contain kept template2") + require.NotContains(t, checksums, orphaned1, "checksum should not contain deleted template") + require.NotContains(t, checksums, orphaned2, "checksum should not contain deleted template") + + // Verify empty directories are purged + require.False(t, fileutil.FolderExists(filepath.Dir(orphaned1)), "empty directory should be purged") + require.False(t, fileutil.FolderExists(filepath.Dir(orphaned2)), "empty directory should be purged") + }) + + t.Run("handles empty templates directory", func(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "nuclei-metadata-empty-test-*") + require.NoError(t, err) + defer func() { + _ = os.RemoveAll(tmpDir) + }() + + cfgdir, err := os.MkdirTemp("", "nuclei-config-*") + require.NoError(t, err) + defer func() { + _ = os.RemoveAll(cfgdir) + }() + + config.DefaultConfig.SetConfigDir(cfgdir) + config.DefaultConfig.SetTemplatesDir(tmpDir) + + // Ensure templates directory exists (even if empty) + require.NoError(t, os.MkdirAll(tmpDir, 0755)) + + // Regenerate metadata on empty directory + err = tm.regenerateTemplateMetadata(tmpDir) + require.NoError(t, err, "should handle empty directory without error") + + // Index should exist but be empty or minimal + indexPath := config.DefaultConfig.GetTemplateIndexFilePath() + if fileutil.FileExists(indexPath) { + index, err := config.GetNucleiTemplatesIndex() + require.NoError(t, err) + require.Empty(t, index, "index should be empty for empty templates directory") + } + }) + + t.Run("purges empty directories", func(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "nuclei-metadata-purge-test-*") + require.NoError(t, err) + defer func() { + _ = os.RemoveAll(tmpDir) + }() + + cfgdir, err := os.MkdirTemp("", "nuclei-config-*") + require.NoError(t, err) + defer func() { + _ = os.RemoveAll(cfgdir) + }() + + config.DefaultConfig.SetConfigDir(cfgdir) + config.DefaultConfig.SetTemplatesDir(tmpDir) + + // Create empty nested directories + emptyDir1 := filepath.Join(tmpDir, "empty1", "nested", "deep") + emptyDir2 := filepath.Join(tmpDir, "empty2") + require.NoError(t, os.MkdirAll(emptyDir1, 0755)) + require.NoError(t, os.MkdirAll(emptyDir2, 0755)) + + // Create one template in a different directory + templateFile := filepath.Join(tmpDir, "kept", "template.yaml") + require.NoError(t, os.MkdirAll(filepath.Dir(templateFile), 0755)) + require.NoError(t, os.WriteFile(templateFile, []byte(`id: kept-template +info: + name: Kept + author: test + severity: info`), 0644)) + + // Regenerate metadata (should purge empty directories) + err = tm.regenerateTemplateMetadata(tmpDir) + require.NoError(t, err) + + // Verify empty directories were purged + require.False(t, fileutil.FolderExists(emptyDir1), "empty nested directory should be purged") + require.False(t, fileutil.FolderExists(emptyDir2), "empty directory should be purged") + require.True(t, fileutil.FolderExists(filepath.Dir(templateFile)), "directory with template should not be purged") + }) +}