过滤非腾讯地址
This commit is contained in:
@@ -20,6 +20,9 @@ from app.models.recruit_announcement_year import RecruitAnnouncementYear
|
|||||||
from app.service.company_service import find_or_create_company
|
from app.service.company_service import find_or_create_company
|
||||||
from app.tool.page_extract import extract_page
|
from app.tool.page_extract import extract_page
|
||||||
|
|
||||||
|
# 微信公众号文章域名,页面提取逻辑只适配了这一种页面结构
|
||||||
|
_WECHAT_DOMAIN = "mp.weixin.qq.com"
|
||||||
|
|
||||||
|
|
||||||
def _parse_datetime(value: str | None) -> datetime | None:
|
def _parse_datetime(value: str | None) -> datetime | None:
|
||||||
"""将 yyyy-MM-dd HH:mm:ss 字符串解析为 datetime,失败返回 None。"""
|
"""将 yyyy-MM-dd HH:mm:ss 字符串解析为 datetime,失败返回 None。"""
|
||||||
@@ -135,7 +138,12 @@ def _save_announcement(announcement_id: int, company_id: int, url: str, data: di
|
|||||||
|
|
||||||
def process_announcement(url: str) -> None:
|
def process_announcement(url: str) -> None:
|
||||||
"""处理单条公告 URL 的完整流程。"""
|
"""处理单条公告 URL 的完整流程。"""
|
||||||
# 1. URL 去重
|
# 1. 只处理微信公众号文章,页面提取逻辑依赖公众号页面结构
|
||||||
|
if _WECHAT_DOMAIN not in url:
|
||||||
|
log.info("非微信公众号文章,跳过: {}", url)
|
||||||
|
return
|
||||||
|
|
||||||
|
# 2. URL 去重
|
||||||
with MysqlSession() as session:
|
with MysqlSession() as session:
|
||||||
row = session.execute(
|
row = session.execute(
|
||||||
text("SELECT id FROM bg_recruit_announcement WHERE announcement_url = :url LIMIT 1"),
|
text("SELECT id FROM bg_recruit_announcement WHERE announcement_url = :url LIMIT 1"),
|
||||||
@@ -145,23 +153,23 @@ def process_announcement(url: str) -> None:
|
|||||||
log.info("公告已存在,跳过: {}", url)
|
log.info("公告已存在,跳过: {}", url)
|
||||||
return
|
return
|
||||||
|
|
||||||
# 2. 页面内容提取
|
# 3. 页面内容提取
|
||||||
result = extract_page(url)
|
result = extract_page(url)
|
||||||
if not result.content or len(result.content) < 50:
|
if not result.content or len(result.content) < 50:
|
||||||
log.info("公告页内容过短({}字),跳过: {}", len(result.content) if result.content else 0, url)
|
log.info("公告页内容过短({}字),跳过: {}", len(result.content) if result.content else 0, url)
|
||||||
return
|
return
|
||||||
|
|
||||||
# 3. AI 信息提取
|
# 4. AI 信息提取
|
||||||
data = extract_announcement(result.content)
|
data = extract_announcement(result.content)
|
||||||
if data is None:
|
if data is None:
|
||||||
log.warning("AI 信息提取失败,跳过: {}", url)
|
log.warning("AI 信息提取失败,跳过: {}", url)
|
||||||
return
|
return
|
||||||
|
|
||||||
# 4. 公司处理
|
# 5. 公司处理
|
||||||
company_name = data.get("company_name") or ""
|
company_name = data.get("company_name") or ""
|
||||||
company_id = find_or_create_company(company_name, result.logo_url)
|
company_id = find_or_create_company(company_name, result.logo_url)
|
||||||
|
|
||||||
# 5. 保存公告
|
# 6. 保存公告
|
||||||
announcement_id = next_id()
|
announcement_id = next_id()
|
||||||
try:
|
try:
|
||||||
_save_announcement(announcement_id, company_id, url, data)
|
_save_announcement(announcement_id, company_id, url, data)
|
||||||
|
|||||||
Reference in New Issue
Block a user