添加初始化数据脚本
This commit is contained in:
@@ -0,0 +1,38 @@
|
||||
"""offershow 存量数据初始化脚本(一次性执行)。
|
||||
|
||||
与定时任务走同一套逻辑,区别仅在于 limit 给得很大,用于首次全量灌数。
|
||||
公告 URL 在 process_announcement 内部会查重,重复执行不会产生脏数据。
|
||||
|
||||
运行(项目根目录下):
|
||||
python init_offershow.py # 用默认 limit
|
||||
python init_offershow.py 500 # 指定本次抓取条数
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
|
||||
from app.core.database import close_db, init_db
|
||||
from app.core.logger import log
|
||||
from app.main import crawl
|
||||
from app.spider.offershow import fetch_offershow
|
||||
|
||||
# 默认抓取条数,接口每页 20 条、翻页上限 2000 页
|
||||
DEFAULT_LIMIT = 5000
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""初始化数据源并跑一轮大批量采集。"""
|
||||
limit = int(sys.argv[1]) if len(sys.argv) > 1 else DEFAULT_LIMIT
|
||||
|
||||
log.info("offershow 存量初始化开始,limit={}", limit)
|
||||
init_db()
|
||||
try:
|
||||
crawl("offershow-init", fetch_offershow, limit)
|
||||
finally:
|
||||
close_db()
|
||||
log.info("offershow 存量初始化结束")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user