"""公告 URL 批量落库:多线程调用单条处理逻辑。""" from __future__ import annotations from concurrent.futures import ThreadPoolExecutor, as_completed from app.config import settings from app.core.logger import log from app.service.recruit_announcement_service import process_announcement def _safe_process(url: str) -> bool: """处理单条 URL,异常不外抛。 Args: url: 公告地址。 Returns: True 表示未抛异常。 """ try: process_announcement(url) return True except Exception as exc: log.error("公告处理异常 [url={}]: {}", url, exc) return False def save_announcements(urls: list[str], workers: int | None = None) -> tuple[int, int]: """多线程处理一批公告 URL。 Args: urls: 公告地址列表,内部去重。 workers: 并发线程数,默认取配置 spider_max_workers。 Returns: (正常条数, 失败条数)。 """ targets = list(dict.fromkeys(u for u in urls if u)) if not targets: return 0, 0 workers = max(1, min(workers or settings.spider_max_workers, len(targets))) log.info("开始处理公告 {} 条,线程数 {}", len(targets), workers) ok = bad = 0 with ThreadPoolExecutor(max_workers=workers, thread_name_prefix="ann") as pool: futures = [pool.submit(_safe_process, url) for url in targets] for future in as_completed(futures): if future.result(): ok += 1 else: bad += 1 log.info("公告处理完成:正常 {} | 失败 {}", ok, bad) return ok, bad