补充文档,修改清洗方案

This commit is contained in:
zk
2026-06-02 18:38:36 +08:00
parent 30e6a6e2a5
commit 6967e4ba54
7 changed files with 363 additions and 80 deletions
+21 -12
View File
@@ -86,6 +86,26 @@ async def _do_clean(data: dict) -> None:
await _update_pg_status(data_id, "discarded")
return
# 哈希去重续命:content_hash 已存在于 bg_job → 说明是重复刷新,直接续命
content_hash = data.get("content_hash")
if content_hash:
async with MysqlSession() as mysql:
row = await mysql.execute(
text("SELECT id FROM bg_job WHERE content_hash = :hash LIMIT 1"),
{"hash": content_hash},
)
existing_id = row.scalar()
if existing_id:
now = datetime.now()
await mysql.execute(
text("UPDATE bg_job SET status = 0, create_time = :now, update_time = :now WHERE id = :id"),
{"now": now, "id": existing_id},
)
await mysql.commit()
await _update_pg_status(data_id, "cleaned")
log.info("[id={}] 续命:hash命中,岗位ID={}", data_id, existing_id)
return
# 第一次AI:结构化提取
user_message = _build_user_message(data)
result = await ai_chat_json(JobCleanModel.STRUCTURE, JOB_STRUCTURE_SYSTEM, user_message)
@@ -94,18 +114,6 @@ async def _do_clean(data: dict) -> None:
await _update_pg_status(data_id, "discarded")
return
# 去重检查
source_id = str(data_id)
async with MysqlSession() as mysql:
existing = await mysql.execute(
text("SELECT COUNT(*) AS cnt FROM bg_job WHERE source_id = :sid"),
{"sid": source_id},
)
if existing.scalar() > 0:
log.info("[id={}] 跳过:已入库(去重)", data_id)
await _update_pg_status(data_id, "cleaned")
return
# 公司处理
company_short_name = result.get("companyShortName") or data.get("company") or ""
company_id = await _find_or_create_company(company_short_name)
@@ -139,6 +147,7 @@ async def _do_clean(data: dict) -> None:
required_industry_id=result.get("requiredIndustryId"),
recruit_category=data.get("recruit_category", 3),
expire_at=data.get("expire_at"),
content_hash=data.get("content_hash"),
source_url=data.get("detail_url"),
source_id=source_id,
status=0,