"""爬虫服务入口:初始化数据库,注册定时任务并启动调度。 运行(必须在项目根目录下以模块方式启动,否则包内 import 找不到 app): python -m app.main """ from __future__ import annotations from apscheduler.schedulers.blocking import BlockingScheduler from app.config import settings from app.core.database import close_db, init_db from app.core.logger import log from app.service.announcement_batch_service import save_announcements from app.spider.offerqingbaoju import fetch_offerqingbaoju from app.spider.offershow import fetch_offershow def crawl(source: str, fetcher, limit: int) -> None: """采集任务:抓公告地址 → 多线程落库。异常不外抛,避免调度器丢任务。 Args: source: 采集源名称,仅用于日志。 fetcher: 爬虫函数,签名 (limit: int) -> list[str]。 limit: 本次抓取条数上限。 """ log.info("[{}] 任务开始,limit={}", source, limit) try: urls = fetcher(limit) if urls: save_announcements(urls) except Exception as exc: log.error("[{}] 任务异常: {}", source, exc) log.info("[{}] 任务结束", source) # 采集源:(名称, 爬虫函数, 抓取条数, 每天执行的时, 分) JOBS = [ ("offershow", fetch_offershow, 100, 0, 30), ("offerqingbaoju", fetch_offerqingbaoju, 100, 2, 36), ] def main() -> None: """初始化数据源并启动定时任务,阻塞运行直到 Ctrl+C。""" log.info("爬虫服务启动,环境={}", settings.env) init_db() scheduler = BlockingScheduler(timezone="Asia/Shanghai") for source, fetcher, limit, hour, minute in JOBS: scheduler.add_job( crawl, "cron", hour=hour, minute=minute, args=(source, fetcher, limit), id=source, # 上一轮没跑完则本轮跳过,防止任务堆叠 max_instances=1, coalesce=True, # 错过触发时间 10 分钟内仍补跑,超过则跳过本次 misfire_grace_time=600, ) log.info("[{}] 已注册,每天 {:02d}:{:02d} 执行", source, hour, minute) try: scheduler.start() except (KeyboardInterrupt, SystemExit): log.info("服务中断,正在退出") finally: close_db() if __name__ == "__main__": main()