98 lines
3.0 KiB
Python
98 lines
3.0 KiB
Python
"""offershow.cn 招聘公告地址抓取。
|
||
|
||
接口说明见 doc/offershow.md（其中"参数必须放 body、匿名恒 10 条"的描述已过时）：
|
||
- 招聘计划列表 `POST /api/od/plan_table`，明文 JSON，无加密，无需登录。
|
||
- 分页参数 `page`/`size` 必须放在 **URL query string**，放 body 无效（恒返回第 1 页）。
|
||
- 业务成功码为 `200001`。
|
||
- 公告地址取每条计划的 `notice_url` 字段。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import httpx
|
||
|
||
from app.core.logger import log
|
||
|
||
# 招聘计划列表接口
|
||
_API_URL = "https://www.offershow.cn/api/od/plan_table"
|
||
|
||
_HEADERS = {
|
||
"accept": "application/json",
|
||
"content-type": "application/json",
|
||
"origin": "https://www.offershow.cn",
|
||
"referer": "https://www.offershow.cn/",
|
||
"user-agent": (
|
||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||
"(KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36"
|
||
),
|
||
}
|
||
|
||
# 业务成功码
|
||
_SUCCESS_CODE = 200001
|
||
|
||
# 每页条数
|
||
_PAGE_SIZE = 20
|
||
|
||
# 请求超时（秒）
|
||
_TIMEOUT = 30
|
||
|
||
# 翻页安全上限，防止 total 异常时死循环
|
||
_MAX_PAGES = 2000
|
||
|
||
|
||
def fetch_offershow(limit: int) -> list[str]:
|
||
"""抓取 offershow 招聘公告地址。
|
||
|
||
Args:
|
||
limit: 抓取条数上限，循环翻页累积直到拿满或翻完。
|
||
|
||
Returns:
|
||
公告地址列表（已去重、过滤空值，保持原始顺序）。
|
||
"""
|
||
if limit <= 0:
|
||
return []
|
||
|
||
urls: list[str] = []
|
||
seen: set[str] = set()
|
||
|
||
with httpx.Client(headers=_HEADERS, timeout=_TIMEOUT) as client:
|
||
for page in range(1, _MAX_PAGES + 1):
|
||
try:
|
||
response = client.post(
|
||
_API_URL,
|
||
params={"page": page, "size": _PAGE_SIZE},
|
||
json={"recruit_plan_type": 2},
|
||
)
|
||
response.raise_for_status()
|
||
payload = response.json()
|
||
except (httpx.HTTPError, ValueError) as exc:
|
||
log.error(f"offershow 抓取失败（page={page}）: {exc}")
|
||
break
|
||
|
||
if payload.get("code") != _SUCCESS_CODE:
|
||
log.warning(f"offershow 返回非成功码（page={page}）: {payload.get('code')}")
|
||
break
|
||
|
||
plans = (payload.get("data") or {}).get("plans") or []
|
||
if not plans:
|
||
break
|
||
|
||
for plan in plans:
|
||
url = (plan.get("notice_url") or "").strip()
|
||
if not url or not url.startswith("http"):
|
||
continue
|
||
if url in seen:
|
||
continue
|
||
seen.add(url)
|
||
urls.append(url)
|
||
if len(urls) >= limit:
|
||
log.info(f"offershow 抓取到 {len(urls)} 条公告地址（翻页 {page} 页）")
|
||
return urls
|
||
|
||
# 已翻到最后一页
|
||
if len(plans) < _PAGE_SIZE:
|
||
break
|
||
|
||
log.info(f"offershow 抓取到 {len(urls)} 条公告地址")
|
||
return urls
|