添加定时任务触发逻辑
This commit is contained in:
+75
@@ -0,0 +1,75 @@
|
||||
"""爬虫服务入口:初始化数据库,注册定时任务并启动调度。
|
||||
|
||||
运行(必须在项目根目录下以模块方式启动,否则包内 import 找不到 app):
|
||||
python -m app.main
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from apscheduler.schedulers.blocking import BlockingScheduler
|
||||
|
||||
from app.config import settings
|
||||
from app.core.database import close_db, init_db
|
||||
from app.core.logger import log
|
||||
from app.service.announcement_batch_service import save_announcements
|
||||
from app.spider.offerqingbaoju import fetch_offerqingbaoju
|
||||
from app.spider.offershow import fetch_offershow
|
||||
|
||||
|
||||
def crawl(source: str, fetcher, limit: int) -> None:
|
||||
"""采集任务:抓公告地址 → 多线程落库。异常不外抛,避免调度器丢任务。
|
||||
|
||||
Args:
|
||||
source: 采集源名称,仅用于日志。
|
||||
fetcher: 爬虫函数,签名 (limit: int) -> list[str]。
|
||||
limit: 本次抓取条数上限。
|
||||
"""
|
||||
log.info("[{}] 任务开始,limit={}", source, limit)
|
||||
try:
|
||||
urls = fetcher(limit)
|
||||
if urls:
|
||||
save_announcements(urls)
|
||||
except Exception as exc:
|
||||
log.error("[{}] 任务异常: {}", source, exc)
|
||||
log.info("[{}] 任务结束", source)
|
||||
|
||||
|
||||
# 采集源:(名称, 爬虫函数, 抓取条数, 每天执行的时, 分)
|
||||
JOBS = [
|
||||
("offershow", fetch_offershow, 100, 0, 30),
|
||||
("offerqingbaoju", fetch_offerqingbaoju, 100, 2, 36),
|
||||
]
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""初始化数据源并启动定时任务,阻塞运行直到 Ctrl+C。"""
|
||||
log.info("爬虫服务启动,环境={}", settings.env)
|
||||
init_db()
|
||||
|
||||
scheduler = BlockingScheduler(timezone="Asia/Shanghai")
|
||||
for source, fetcher, limit, hour, minute in JOBS:
|
||||
scheduler.add_job(
|
||||
crawl,
|
||||
"cron",
|
||||
hour=hour,
|
||||
minute=minute,
|
||||
args=(source, fetcher, limit),
|
||||
id=source,
|
||||
# 上一轮没跑完则本轮跳过,防止任务堆叠
|
||||
max_instances=1,
|
||||
coalesce=True,
|
||||
# 错过触发时间 10 分钟内仍补跑,超过则跳过本次
|
||||
misfire_grace_time=600,
|
||||
)
|
||||
log.info("[{}] 已注册,每天 {:02d}:{:02d} 执行", source, hour, minute)
|
||||
|
||||
try:
|
||||
scheduler.start()
|
||||
except (KeyboardInterrupt, SystemExit):
|
||||
log.info("服务中断,正在退出")
|
||||
finally:
|
||||
close_db()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user