57 lines
1.6 KiB
Python
57 lines
1.6 KiB
Python
"""公告 URL 批量落库：多线程调用单条处理逻辑。"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
|
|
from app.config import settings
|
|
from app.core.logger import log
|
|
from app.service.recruit_announcement_service import process_announcement
|
|
|
|
|
|
def _safe_process(url: str) -> bool:
|
|
"""处理单条 URL，异常不外抛。
|
|
|
|
Args:
|
|
url: 公告地址。
|
|
|
|
Returns:
|
|
True 表示未抛异常。
|
|
"""
|
|
try:
|
|
process_announcement(url)
|
|
return True
|
|
except Exception as exc:
|
|
log.error("公告处理异常 [url={}]: {}", url, exc)
|
|
return False
|
|
|
|
|
|
def save_announcements(urls: list[str], workers: int | None = None) -> tuple[int, int]:
|
|
"""多线程处理一批公告 URL。
|
|
|
|
Args:
|
|
urls: 公告地址列表，内部去重。
|
|
workers: 并发线程数，默认取配置 spider_max_workers。
|
|
|
|
Returns:
|
|
(正常条数, 失败条数)。
|
|
"""
|
|
targets = list(dict.fromkeys(u for u in urls if u))
|
|
if not targets:
|
|
return 0, 0
|
|
|
|
workers = max(1, min(workers or settings.spider_max_workers, len(targets)))
|
|
log.info("开始处理公告 {} 条，线程数 {}", len(targets), workers)
|
|
|
|
ok = bad = 0
|
|
with ThreadPoolExecutor(max_workers=workers, thread_name_prefix="ann") as pool:
|
|
futures = [pool.submit(_safe_process, url) for url in targets]
|
|
for future in as_completed(futures):
|
|
if future.result():
|
|
ok += 1
|
|
else:
|
|
bad += 1
|
|
|
|
log.info("公告处理完成：正常 {} | 失败 {}", ok, bad)
|
|
return ok, bad
|