76 lines
2.4 KiB
Python
76 lines
2.4 KiB
Python
"""爬虫服务入口：初始化数据库，注册定时任务并启动调度。
|
|
|
|
运行（必须在项目根目录下以模块方式启动，否则包内 import 找不到 app）：
|
|
python -m app.main
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from apscheduler.schedulers.blocking import BlockingScheduler
|
|
|
|
from app.config import settings
|
|
from app.core.database import close_db, init_db
|
|
from app.core.logger import log
|
|
from app.service.announcement_batch_service import save_announcements
|
|
from app.spider.offerqingbaoju import fetch_offerqingbaoju
|
|
from app.spider.offershow import fetch_offershow
|
|
|
|
|
|
def crawl(source: str, fetcher, limit: int) -> None:
|
|
"""采集任务：抓公告地址 → 多线程落库。异常不外抛，避免调度器丢任务。
|
|
|
|
Args:
|
|
source: 采集源名称，仅用于日志。
|
|
fetcher: 爬虫函数，签名 (limit: int) -> list[str]。
|
|
limit: 本次抓取条数上限。
|
|
"""
|
|
log.info("[{}] 任务开始，limit={}", source, limit)
|
|
try:
|
|
urls = fetcher(limit)
|
|
if urls:
|
|
save_announcements(urls)
|
|
except Exception as exc:
|
|
log.error("[{}] 任务异常: {}", source, exc)
|
|
log.info("[{}] 任务结束", source)
|
|
|
|
|
|
# 采集源：(名称, 爬虫函数, 抓取条数, 每天执行的时, 分)
|
|
JOBS = [
|
|
("offershow", fetch_offershow, 100, 0, 30),
|
|
("offerqingbaoju", fetch_offerqingbaoju, 100, 2, 36),
|
|
]
|
|
|
|
|
|
def main() -> None:
|
|
"""初始化数据源并启动定时任务，阻塞运行直到 Ctrl+C。"""
|
|
log.info("爬虫服务启动，环境={}", settings.env)
|
|
init_db()
|
|
|
|
scheduler = BlockingScheduler(timezone="Asia/Shanghai")
|
|
for source, fetcher, limit, hour, minute in JOBS:
|
|
scheduler.add_job(
|
|
crawl,
|
|
"cron",
|
|
hour=hour,
|
|
minute=minute,
|
|
args=(source, fetcher, limit),
|
|
id=source,
|
|
# 上一轮没跑完则本轮跳过，防止任务堆叠
|
|
max_instances=1,
|
|
coalesce=True,
|
|
# 错过触发时间 10 分钟内仍补跑，超过则跳过本次
|
|
misfire_grace_time=600,
|
|
)
|
|
log.info("[{}] 已注册，每天 {:02d}:{:02d} 执行", source, hour, minute)
|
|
|
|
try:
|
|
scheduler.start()
|
|
except (KeyboardInterrupt, SystemExit):
|
|
log.info("服务中断，正在退出")
|
|
finally:
|
|
close_db()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|