diff --git a/app/spider/__init__.py b/app/spider/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/spider/offerqingbaoju.py b/app/spider/offerqingbaoju.py new file mode 100644 index 0000000..4345489 --- /dev/null +++ b/app/spider/offerqingbaoju.py @@ -0,0 +1,69 @@ +"""offerqingbaoju.cn 招聘公告地址抓取。 + +接口说明见 doc/offerqingbaoju.md: +- 明文 JSON,无加密。 +- 匿名访问只能取 page=1,但 per_page 可以拉很大(实测 6000+ 条一次返回)。 +- 公告地址取每条数据的 `公告链接` 字段。 +""" + +from __future__ import annotations + +import httpx + +from app.core.logger import log + +# 招聘数据接口 +_API_URL = "https://offerqingbaoju.cn/api/simple/navigation/60/data" + +_HEADERS = { + "accept": "application/json", + "user-agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36" + ), +} + +# 请求超时(秒) +_TIMEOUT = 30 + + +def fetch_offerqingbaoju(limit: int) -> list[str]: + """抓取 offerqingbaoju 招聘公告地址。 + + Args: + limit: 抓取条数(映射到接口的 per_page,匿名只取第 1 页)。 + + Returns: + 公告地址列表(已去重、过滤空值,保持原始顺序)。 + """ + if limit <= 0: + return [] + + try: + response = httpx.get( + _API_URL, + params={"page": 1, "per_page": limit}, + headers=_HEADERS, + timeout=_TIMEOUT, + ) + response.raise_for_status() + payload = response.json() + except (httpx.HTTPError, ValueError) as exc: + log.error(f"offerqingbaoju 抓取失败: {exc}") + return [] + + rows = payload.get("data") or [] + + urls: list[str] = [] + seen: set[str] = set() + for row in rows: + url = (row.get("公告链接") or "").strip() + if not url or not url.startswith("http"): + continue + if url in seen: + continue + seen.add(url) + urls.append(url) + + log.info(f"offerqingbaoju 抓取到 {len(urls)} 条公告地址(原始 {len(rows)} 条)") + return urls diff --git a/app/spider/offershow.py b/app/spider/offershow.py new file mode 100644 index 0000000..f370242 --- /dev/null +++ b/app/spider/offershow.py @@ -0,0 +1,97 @@ +"""offershow.cn 招聘公告地址抓取。 + +接口说明见 doc/offershow.md(其中"参数必须放 body、匿名恒 10 条"的描述已过时): +- 招聘计划列表 `POST /api/od/plan_table`,明文 JSON,无加密,无需登录。 +- 分页参数 `page`/`size` 必须放在 **URL query string**,放 body 无效(恒返回第 1 页)。 +- 业务成功码为 `200001`。 +- 公告地址取每条计划的 `notice_url` 字段。 +""" + +from __future__ import annotations + +import httpx + +from app.core.logger import log + +# 招聘计划列表接口 +_API_URL = "https://www.offershow.cn/api/od/plan_table" + +_HEADERS = { + "accept": "application/json", + "content-type": "application/json", + "origin": "https://www.offershow.cn", + "referer": "https://www.offershow.cn/", + "user-agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36" + ), +} + +# 业务成功码 +_SUCCESS_CODE = 200001 + +# 每页条数 +_PAGE_SIZE = 20 + +# 请求超时(秒) +_TIMEOUT = 30 + +# 翻页安全上限,防止 total 异常时死循环 +_MAX_PAGES = 2000 + + +def fetch_offershow(limit: int) -> list[str]: + """抓取 offershow 招聘公告地址。 + + Args: + limit: 抓取条数上限,循环翻页累积直到拿满或翻完。 + + Returns: + 公告地址列表(已去重、过滤空值,保持原始顺序)。 + """ + if limit <= 0: + return [] + + urls: list[str] = [] + seen: set[str] = set() + + with httpx.Client(headers=_HEADERS, timeout=_TIMEOUT) as client: + for page in range(1, _MAX_PAGES + 1): + try: + response = client.post( + _API_URL, + params={"page": page, "size": _PAGE_SIZE}, + json={"recruit_plan_type": 2}, + ) + response.raise_for_status() + payload = response.json() + except (httpx.HTTPError, ValueError) as exc: + log.error(f"offershow 抓取失败(page={page}): {exc}") + break + + if payload.get("code") != _SUCCESS_CODE: + log.warning(f"offershow 返回非成功码(page={page}): {payload.get('code')}") + break + + plans = (payload.get("data") or {}).get("plans") or [] + if not plans: + break + + for plan in plans: + url = (plan.get("notice_url") or "").strip() + if not url or not url.startswith("http"): + continue + if url in seen: + continue + seen.add(url) + urls.append(url) + if len(urls) >= limit: + log.info(f"offershow 抓取到 {len(urls)} 条公告地址(翻页 {page} 页)") + return urls + + # 已翻到最后一页 + if len(plans) < _PAGE_SIZE: + break + + log.info(f"offershow 抓取到 {len(urls)} 条公告地址") + return urls diff --git a/requirements.txt b/requirements.txt index 51510c5..aa7f0f8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ cryptography>=41 +httpx>=0.28 rapidocr>=3.9 onnxruntime>=1.17 opencv-python>=4.5