mirror of
https://github.com/yyhuni/xingrin.git
synced 2026-02-12 17:33:36 +08:00
28 lines
937 B
Python
28 lines
937 B
Python
"""
|
|
URL 获取任务模块
|
|
|
|
包含 URL 获取相关的所有原子任务：
|
|
- export_sites_task: 导出站点 URL 列表（用于 katana 等爬虫）
|
|
- run_url_fetcher_task: 执行 URL 获取工具
|
|
- merge_and_deduplicate_urls_task: 合并去重 URL
|
|
- clean_urls_task: 使用 uro 清理 URL
|
|
- save_urls_task: 保存 URL 到数据库
|
|
- run_and_stream_save_urls_task: 流式验证并保存存活的 URL
|
|
"""
|
|
|
|
from .export_sites_task import export_sites_task
|
|
from .run_url_fetcher_task import run_url_fetcher_task
|
|
from .merge_and_deduplicate_urls_task import merge_and_deduplicate_urls_task
|
|
from .clean_urls_task import clean_urls_task
|
|
from .save_urls_task import save_urls_task
|
|
from .run_and_stream_save_urls_task import run_and_stream_save_urls_task
|
|
|
|
__all__ = [
|
|
'export_sites_task',
|
|
'run_url_fetcher_task',
|
|
'merge_and_deduplicate_urls_task',
|
|
'clean_urls_task',
|
|
'save_urls_task',
|
|
'run_and_stream_save_urls_task',
|
|
]
|