补充文档,修改清洗方案
This commit is contained in:
@@ -86,6 +86,26 @@ async def _do_clean(data: dict) -> None:
|
||||
await _update_pg_status(data_id, "discarded")
|
||||
return
|
||||
|
||||
# 哈希去重续命:content_hash 已存在于 bg_job → 说明是重复刷新,直接续命
|
||||
content_hash = data.get("content_hash")
|
||||
if content_hash:
|
||||
async with MysqlSession() as mysql:
|
||||
row = await mysql.execute(
|
||||
text("SELECT id FROM bg_job WHERE content_hash = :hash LIMIT 1"),
|
||||
{"hash": content_hash},
|
||||
)
|
||||
existing_id = row.scalar()
|
||||
if existing_id:
|
||||
now = datetime.now()
|
||||
await mysql.execute(
|
||||
text("UPDATE bg_job SET status = 0, create_time = :now, update_time = :now WHERE id = :id"),
|
||||
{"now": now, "id": existing_id},
|
||||
)
|
||||
await mysql.commit()
|
||||
await _update_pg_status(data_id, "cleaned")
|
||||
log.info("[id={}] 续命:hash命中,岗位ID={}", data_id, existing_id)
|
||||
return
|
||||
|
||||
# 第一次AI:结构化提取
|
||||
user_message = _build_user_message(data)
|
||||
result = await ai_chat_json(JobCleanModel.STRUCTURE, JOB_STRUCTURE_SYSTEM, user_message)
|
||||
@@ -94,18 +114,6 @@ async def _do_clean(data: dict) -> None:
|
||||
await _update_pg_status(data_id, "discarded")
|
||||
return
|
||||
|
||||
# 去重检查
|
||||
source_id = str(data_id)
|
||||
async with MysqlSession() as mysql:
|
||||
existing = await mysql.execute(
|
||||
text("SELECT COUNT(*) AS cnt FROM bg_job WHERE source_id = :sid"),
|
||||
{"sid": source_id},
|
||||
)
|
||||
if existing.scalar() > 0:
|
||||
log.info("[id={}] 跳过:已入库(去重)", data_id)
|
||||
await _update_pg_status(data_id, "cleaned")
|
||||
return
|
||||
|
||||
# 公司处理
|
||||
company_short_name = result.get("companyShortName") or data.get("company") or ""
|
||||
company_id = await _find_or_create_company(company_short_name)
|
||||
@@ -139,6 +147,7 @@ async def _do_clean(data: dict) -> None:
|
||||
required_industry_id=result.get("requiredIndustryId"),
|
||||
recruit_category=data.get("recruit_category", 3),
|
||||
expire_at=data.get("expire_at"),
|
||||
content_hash=data.get("content_hash"),
|
||||
source_url=data.get("detail_url"),
|
||||
source_id=source_id,
|
||||
status=0,
|
||||
|
||||
Reference in New Issue
Block a user