初始化

app/services/ai_tool.py

@@ -0,0 +1,65 @@
+"""AI 调用工具封装"""
+
+import json
+import re
+from typing import Any
+
+from langchain_openai import ChatOpenAI
+from langchain_core.messages import SystemMessage, HumanMessage
+
+from app.core.logger import log
+
+# markdown 代码块正则
+_CODE_BLOCK_RE = re.compile(r"```\w*\s*\n?(.*?)\n?\s*```", re.DOTALL)
+# 控制字符正则（保留 \t \n \r）
+_CONTROL_CHAR_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f]")
+
+
+def clean_ai_response(response: str) -> str:
+    """从 AI 返回的文本中提取干净的 JSON 字符串"""
+    if not response or not response.strip():
+        return ""
+
+    result = response.strip()
+
+    # 尝试从 markdown 代码块提取
+    match = _CODE_BLOCK_RE.search(result)
+    if match:
+        result = match.group(1).strip()
+    else:
+        # 定位首个 JSON 起始符
+        obj_start = result.find("{")
+        arr_start = result.find("[")
+        if obj_start < 0:
+            start = arr_start
+        elif arr_start < 0:
+            start = obj_start
+        else:
+            start = min(obj_start, arr_start)
+        if start > 0:
+            result = result[start:]
+
+    # 清除控制字符
+    result = _CONTROL_CHAR_RE.sub("", result)
+    return result
+
+
+async def ai_chat(llm: ChatOpenAI, system_prompt: str, user_message: str) -> str:
+    """异步调用 LLM，返回原始文本"""
+    messages = [
+        SystemMessage(content=system_prompt),
+        HumanMessage(content=user_message),
+    ]
+    response = await llm.ainvoke(messages)
+    return response.content
+
+
+async def ai_chat_json(llm: ChatOpenAI, system_prompt: str, user_message: str) -> Any:
+    """异步调用 LLM，返回解析后的 JSON 对象"""
+    raw = await ai_chat(llm, system_prompt, user_message)
+    cleaned = clean_ai_response(raw)
+    try:
+        return json.loads(cleaned)
+    except json.JSONDecodeError as e:
+        log.warning("AI JSON 解析失败: {}, raw={}", e, raw[:200])
+        return None

app/services/company_clean_service.py

@@ -0,0 +1,138 @@
+"""公司数据补充服务（协程版）"""
+
+import asyncio
+from datetime import datetime
+
+from sqlalchemy import text
+
+from app.config import settings
+from app.core.database import MysqlSession
+from app.core.logger import log
+from app.ai.model_config import CompanyCleanModel
+from app.ai.prompts import COMPANY_ENRICH_SYSTEM
+from app.services.ai_tool import ai_chat_json
+from app.services.dict_cache_service import dict_cache
+
+
+async def run_company_clean() -> None:
+    """一次批量公司补充任务"""
+    # 锁定一批待完善公司
+    async with MysqlSession() as mysql:
+        result = await mysql.execute(
+            text("""
+                SELECT * FROM bg_company
+                WHERE status = 0
+                LIMIT :limit
+                FOR UPDATE SKIP LOCKED
+            """),
+            {"limit": settings.company_batch_size},
+        )
+        rows = result.mappings().all()
+        if not rows:
+            return
+
+        ids = [r["id"] for r in rows]
+        # MySQL 批量 IN 用 format 拼接（id 是 bigint，安全）
+        ids_str = ",".join(str(i) for i in ids)
+        await mysql.execute(
+            text(f"UPDATE bg_company SET status = 3, update_time = NOW() WHERE id IN ({ids_str})"),
+        )
+        await mysql.commit()
+
+    log.info("公司补充：锁定{}条数据", len(rows))
+
+    # 协程并发，信号量限流
+    sem = asyncio.Semaphore(settings.company_concurrency)
+    tasks = [_clean_one(sem, dict(r)) for r in rows]
+    await asyncio.gather(*tasks, return_exceptions=True)
+
+
+async def _clean_one(sem: asyncio.Semaphore, company: dict) -> None:
+    """单条公司补充"""
+    async with sem:
+        try:
+            await _do_clean(company)
+        except Exception as e:
+            log.error("公司补充异常, id={}, shortName={}: {}", company["id"], company.get("short_name"), e)
+
+
+async def _do_clean(company: dict) -> None:
+    """公司补充逻辑"""
+    company_id = company["id"]
+    short_name = company.get("short_name", "")
+
+    user_msg = f"【公司简称】\n{short_name}\n\n【行业列表】\n{dict_cache.industry_text}"
+    result = await ai_chat_json(CompanyCleanModel.ENRICH, COMPANY_ENRICH_SYSTEM, user_msg)
+
+    if result is None or not result.get("valid", False):
+        await _update_status(company_id, 4)
+        return
+
+    # 地区匹配
+    city = result.get("city")
+    region_code = dict_cache.match_region_code(city) if city else None
+
+    # 回填数据
+    now = datetime.now()
+    async with MysqlSession() as mysql:
+        await mysql.execute(
+            text("""
+                UPDATE bg_company SET
+                    name = COALESCE(:name, name),
+                    region_code = COALESCE(:region_code, region_code),
+                    company_type = COALESCE(:company_type, company_type),
+                    industry_id = :industry_id,
+                    tags = :tags,
+                    summary = COALESCE(:summary, summary),
+                    description = COALESCE(:description, description),
+                    founded_year = COALESCE(:founded_year, founded_year),
+                    address = COALESCE(:address, address),
+                    scale = COALESCE(:scale, scale),
+                    website = COALESCE(:website, website),
+                    financing_stage = COALESCE(:financing_stage, financing_stage),
+                    latest_valuation = COALESCE(:latest_valuation, latest_valuation),
+                    news = :news,
+                    status = 1,
+                    update_time = :now
+                WHERE id = :id
+            """),
+            {
+                "name": result.get("name"),
+                "region_code": region_code,
+                "company_type": result.get("companyType"),
+                "industry_id": result.get("industryId"),
+                "tags": _to_json(result.get("tags")),
+                "summary": result.get("summary"),
+                "description": result.get("description"),
+                "founded_year": result.get("foundedYear"),
+                "address": result.get("address"),
+                "scale": result.get("scale"),
+                "website": result.get("website"),
+                "financing_stage": result.get("financingStage"),
+                "latest_valuation": result.get("latestValuation"),
+                "news": _to_json(result.get("news")),
+                "now": now,
+                "id": company_id,
+            },
+        )
+        await mysql.commit()
+
+    log.info("公司补充完成, id={}, shortName={}", company_id, short_name)
+
+
+async def _update_status(company_id: int, status: int) -> None:
+    """更新公司状态"""
+    async with MysqlSession() as mysql:
+        await mysql.execute(
+            text("UPDATE bg_company SET status = :s, update_time = NOW() WHERE id = :id"),
+            {"s": status, "id": company_id},
+        )
+        await mysql.commit()
+
+
+def _to_json(value) -> str | None:
+    """列表转 JSON 字符串"""
+    import json
+    if value and isinstance(value, list):
+        return json.dumps(value, ensure_ascii=False)
+    return None

app/services/dict_cache_service.py

@@ -0,0 +1,91 @@
+"""字典数据缓存服务
+
+启动时从 MySQL 加载岗位分类、行业、专业分类、地区数据到内存。
+"""
+
+from sqlalchemy import select, text
+
+from app.core.database import MysqlSession
+from app.core.logger import log
+
+
+class DictCacheService:
+    """字典缓存，单例使用"""
+
+    def __init__(self):
+        self.job_category_text: str = ""
+        self.industry_text: str = ""
+        self.major_category_text: str = ""
+        self._region_list: list[dict] = []
+
+    async def refresh(self) -> None:
+        """加载全量字典数据"""
+        async with MysqlSession() as session:
+            # 岗位分类（三级叶子，带父级路径）
+            result = await session.execute(text("""
+                SELECT c.id, c.name, c.parent_id, c.root_id, c.level,
+                       p.name AS parent_name, r.name AS root_name
+                FROM bg_job_category c
+                LEFT JOIN bg_job_category p ON c.parent_id = p.id
+                LEFT JOIN bg_job_category r ON c.root_id = r.id
+                WHERE c.level = 3
+            """))
+            categories = result.mappings().all()
+            self.job_category_text = ", ".join(
+                f"{c['id']}:{c['name']}({c['root_name']}/{c['parent_name']})"
+                for c in categories
+            )
+
+            # 行业（二级叶子，带父级）
+            result = await session.execute(text("""
+                SELECT i.id, i.name, p.name AS parent_name
+                FROM bg_industry i
+                LEFT JOIN bg_industry p ON i.parent_id = p.id
+                WHERE i.level = 2
+            """))
+            industries = result.mappings().all()
+            self.industry_text = ", ".join(
+                f"{i['id']}:{i['name']}({i['parent_name']})"
+                for i in industries
+            )
+
+            # 专业分类（三级叶子，带父级路径）
+            result = await session.execute(text("""
+                SELECT m.id, m.name, m.parent_id, m.root_id,
+                       p.name AS parent_name, r.name AS root_name
+                FROM bg_major_category m
+                LEFT JOIN bg_major_category p ON m.parent_id = p.id
+                LEFT JOIN bg_major_category r ON m.root_id = r.id
+                WHERE m.level = 3
+            """))
+            majors = result.mappings().all()
+            self.major_category_text = ", ".join(
+                f"{m['id']}:{m['name']}({m['root_name']}/{m['parent_name']})"
+                for m in majors
+            )
+
+            # 地区（省市级）
+            result = await session.execute(text("""
+                SELECT code, name FROM bg_china_regions_code WHERE city_code IS NULL
+            """))
+            self._region_list = [dict(r) for r in result.mappings().all()]
+
+        log.info(
+            "字典缓存加载完成: 岗位分类{}条, 行业{}条, 专业{}条, 地区{}条",
+            len(categories), len(industries), len(majors), len(self._region_list),
+        )
+
+    def match_region_code(self, city_name: str) -> str | None:
+        """根据城市名模糊匹配地区编码"""
+        if not city_name:
+            return None
+        name = city_name.replace("市", "").replace("省", "").strip()
+        for r in self._region_list:
+            r_name = r["name"].replace("市", "").replace("省", "")
+            if name in r_name or r_name in name:
+                return r["code"]
+        return None
+
+
+# 全局单例
+dict_cache = DictCacheService()

app/services/job_clean_service.py

@@ -0,0 +1,306 @@
+"""岗位清洗服务（协程版）"""
+
+import asyncio
+import json
+from datetime import datetime
+
+from snowflake import SnowflakeGenerator
+from sqlalchemy import text, insert
+
+from app.config import settings
+from app.core.database import PgSession, MysqlSession
+from app.core.logger import log
+from app.ai.model_config import JobCleanModel
+from app.ai.prompts import JOB_STRUCTURE_SYSTEM, MAJOR_MATCH_SYSTEM, SKILL_EXTRACT_SYSTEM
+from app.models.mysql.job import Job
+from app.models.mysql.company import Company
+from app.models.mysql.relations import JobRegionRelation, JobSkillTagRelation
+from app.services.ai_tool import ai_chat_json
+from app.services.dict_cache_service import dict_cache
+
+# 雪花ID生成器
+_id_gen = SnowflakeGenerator(instance=1)
+
+# 公司创建锁（防止并发重复插入同一公司）
+_company_lock = asyncio.Lock()
+
+
+async def run_job_clean() -> None:
+    """一次批量清洗任务"""
+    # 1. 从 PG 锁定一批待清洗数据
+    async with PgSession() as pg:
+        result = await pg.execute(
+            text("""
+                SELECT * FROM app_job_data
+                WHERE clean_status = 'pending'
+                LIMIT :limit
+                FOR UPDATE SKIP LOCKED
+            """),
+            {"limit": settings.clean_batch_size},
+        )
+        rows = result.mappings().all()
+        if not rows:
+            return
+
+        ids = [r["id"] for r in rows]
+        await pg.execute(
+            text("""
+                UPDATE app_job_data
+                SET clean_status = 'cleaning', clean_started_at = NOW()
+                WHERE id = ANY(:ids)
+            """),
+            {"ids": ids},
+        )
+        await pg.commit()
+
+    log.info("岗位清洗：锁定{}条数据", len(rows))
+
+    # 2. 协程并发清洗，信号量限流
+    sem = asyncio.Semaphore(settings.clean_concurrency)
+    tasks = [_clean_one(sem, dict(r)) for r in rows]
+    results = await asyncio.gather(*tasks, return_exceptions=True)
+
+    # 汇总
+    errors = sum(1 for r in results if isinstance(r, Exception))
+    log.info("岗位清洗：本批完成，共{}条，异常{}条", len(rows), errors)
+
+
+async def _clean_one(sem: asyncio.Semaphore, data: dict) -> None:
+    """单条岗位清洗"""
+    async with sem:
+        try:
+            await _do_clean(data)
+        except Exception as e:
+            log.error("岗位清洗异常, id={}: {}", data["id"], e)
+            # 保持 cleaning 状态，由僵尸恢复任务重置
+
+
+async def _do_clean(data: dict) -> None:
+    """清洗逻辑"""
+    data_id = data["id"]
+
+    # 前置校验
+    description = data.get("description") or ""
+    if len(description) < 20:
+        log.info("[id={}] 丢弃：描述过短({}字符)", data_id, len(description))
+        await _update_pg_status(data_id, "discarded")
+        return
+
+    # 第一次AI：结构化提取
+    user_message = _build_user_message(data)
+    result = await ai_chat_json(JobCleanModel.STRUCTURE, JOB_STRUCTURE_SYSTEM, user_message)
+    if result is None or not result.get("valid", False):
+        log.info("[id={}] 丢弃：AI判定无效", data_id)
+        await _update_pg_status(data_id, "discarded")
+        return
+
+    # 去重检查
+    source_id = str(data_id)
+    async with MysqlSession() as mysql:
+        existing = await mysql.execute(
+            text("SELECT COUNT(*) AS cnt FROM bg_job WHERE source_id = :sid"),
+            {"sid": source_id},
+        )
+        if existing.scalar() > 0:
+            log.info("[id={}] 跳过：已入库(去重)", data_id)
+            await _update_pg_status(data_id, "cleaned")
+            return
+
+    # 公司处理
+    company_short_name = result.get("companyShortName") or data.get("company") or ""
+    company_id = await _find_or_create_company(company_short_name)
+
+    # 地区处理
+    region_codes = []
+    for city in result.get("cities") or []:
+        code = dict_cache.match_region_code(city)
+        if code:
+            region_codes.append(code)
+
+    # 写入 bg_job
+    job_id = next(_id_gen)
+    now = datetime.now()
+    async with MysqlSession() as mysql:
+        await mysql.execute(
+            insert(Job).values(
+                id=job_id,
+                title=result.get("title", ""),
+                company_id=company_id,
+                category_id=result.get("categoryId", 0),
+                employment_type=result.get("employmentType", 0),
+                description=result.get("description", ""),
+                requirement=result.get("requirement", ""),
+                bonus=result.get("bonus"),
+                tags=result.get("tags"),
+                skill_tags=result.get("skillTags"),
+                salary=result.get("salary"),
+                education=result.get("education", 0),
+                min_experience=result.get("minExperience", 0),
+                required_industry_id=result.get("requiredIndustryId"),
+                recruit_category=data.get("recruit_category", 3),
+                expire_at=data.get("expire_at"),
+                source_url=data.get("detail_url"),
+                source_id=source_id,
+                status=0,
+                create_time=now,
+                update_time=now,
+            )
+        )
+
+        # 写入地区关联
+        if region_codes:
+            await mysql.execute(
+                insert(JobRegionRelation),
+                [{"id": next(_id_gen), "job_id": job_id, "region_code": code, "create_time": now} for code in region_codes],
+            )
+
+        await mysql.commit()
+
+    # 更新 PG 状态
+    await _update_pg_status(data_id, "cleaned")
+    log.info("[id={}] 入库成功：{} | 公司={} | 地区={}", data_id, result.get("title"), company_short_name, region_codes)
+
+    # 第二次AI：专业匹配（失败不影响）
+    try:
+        await _match_major(job_id, result)
+        log.debug("[id={}] 专业匹配完成", data_id)
+    except Exception as e:
+        log.warning("[id={}] 专业匹配失败: {}", data_id, e)
+
+    # 第三次AI：技能提取（失败不影响）
+    try:
+        await _extract_skill_tags(job_id, result)
+        log.debug("[id={}] 技能提取完成", data_id)
+    except Exception as e:
+        log.warning("[id={}] 技能提取失败: {}", data_id, e)
+
+
+async def _match_major(job_id: int, result: dict) -> None:
+    """第二次AI：专业匹配"""
+    title = result.get("title", "")
+    desc = result.get("description", "")
+    req = result.get("requirement", "")
+    user_msg = f"【岗位信息】\n标题: {title}\n职责: {desc}\n要求: {req}\n\n【专业分类列表】\n{dict_cache.major_category_text}"
+
+    data = await ai_chat_json(JobCleanModel.MAJOR_MATCH, MAJOR_MATCH_SYSTEM, user_msg)
+    if data is None:
+        return
+
+    major_ids = [mid for mid in (data.get("requiredMajorIds") or []) if mid > 0]
+    sensitivity = data.get("majorSensitivity", 0)
+
+    async with MysqlSession() as mysql:
+        await mysql.execute(
+            text("""
+                UPDATE bg_job SET required_major_ids = :ids, major_sensitivity = :s, update_time = :t
+                WHERE id = :jid
+            """),
+            {"ids": json.dumps(major_ids) if major_ids else None, "s": sensitivity, "t": datetime.now(), "jid": job_id},
+        )
+        await mysql.commit()
+
+
+async def _extract_skill_tags(job_id: int, result: dict) -> None:
+    """第三次AI：技能提取"""
+    title = result.get("title", "")
+    desc = result.get("description", "")
+    req = result.get("requirement", "")
+    user_msg = f"【岗位信息】\n标题: {title}\n职责: {desc}\n要求: {req}"
+
+    skills = await ai_chat_json(JobCleanModel.SKILL_EXTRACT, SKILL_EXTRACT_SYSTEM, user_msg)
+    if not skills or not isinstance(skills, list):
+        return
+
+    now = datetime.now()
+    tag_ids = []
+    async with MysqlSession() as mysql:
+        for name in skills:
+            name = str(name).strip().lower()
+            if not name or len(name) > 50:
+                continue
+
+            tag_id = next(_id_gen)
+            # INSERT IGNORE
+            await mysql.execute(
+                text("INSERT IGNORE INTO bg_skill_tag (id, name) VALUES (:id, :name)"),
+                {"id": tag_id, "name": name},
+            )
+            # 查回真实ID
+            row = await mysql.execute(
+                text("SELECT id FROM bg_skill_tag WHERE name = :name LIMIT 1"),
+                {"name": name},
+            )
+            real_id = row.scalar()
+            if real_id and real_id not in tag_ids:
+                tag_ids.append(real_id)
+
+        if tag_ids:
+            await mysql.execute(
+                insert(JobSkillTagRelation),
+                [{"id": next(_id_gen), "job_id": job_id, "skill_tag_id": tid, "create_time": now} for tid in tag_ids],
+            )
+
+        await mysql.commit()
+
+
+async def _find_or_create_company(short_name: str) -> int:
+    """查找或创建公司（加锁防并发重复）"""
+    async with _company_lock:
+        async with MysqlSession() as mysql:
+            row = await mysql.execute(
+                text("SELECT id FROM bg_company WHERE short_name = :name LIMIT 1"),
+                {"name": short_name},
+            )
+            existing = row.scalar()
+            if existing:
+                return existing
+
+            company_id = next(_id_gen)
+            now = datetime.now()
+            await mysql.execute(
+                insert(Company).values(
+                    id=company_id,
+                    name=short_name,
+                    short_name=short_name,
+                    status=0,
+                    create_time=now,
+                    update_time=now,
+                )
+            )
+            await mysql.commit()
+            return company_id
+
+
+async def _update_pg_status(data_id: int, status: str) -> None:
+    """更新 PG 清洗状态"""
+    async with PgSession() as pg:
+        if status == "cleaned":
+            await pg.execute(
+                text("UPDATE app_job_data SET clean_status = :s, cleaned_at = NOW() WHERE id = :id"),
+                {"s": status, "id": data_id},
+            )
+        else:
+            await pg.execute(
+                text("UPDATE app_job_data SET clean_status = :s WHERE id = :id"),
+                {"s": status, "id": data_id},
+            )
+        await pg.commit()
+
+
+def _build_user_message(data: dict) -> str:
+    """构建第一次AI的用户消息"""
+    parts = [
+        "【原始数据】",
+        f"岗位名称: {data.get('job_title') or ''}",
+        f"薪资: {data.get('salary') or ''}",
+        f"工作地点: {data.get('location') or ''}",
+        f"公司: {data.get('company') or ''}",
+        f"经验要求: {data.get('experience') or ''}",
+        f"学历要求: {data.get('education') or ''}",
+        f"岗位详情: {data.get('description') or ''}",
+        "",
+        f"【岗位分类列表】\n{dict_cache.job_category_text}",
+        "",
+        f"【行业列表】\n{dict_cache.industry_text}",
+    ]
+    return "\n".join(parts)

app/services/job_expire_service.py

@@ -0,0 +1,29 @@
+"""岗位下架服务
+
+每天定时执行，将 create_time 超过 N 天的岗位标记为已失效。
+"""
+
+from sqlalchemy import text
+
+from app.config import settings
+from app.core.database import MysqlSession
+from app.core.logger import log
+
+
+async def run_job_expire() -> None:
+    """下架过期岗位"""
+    days = int(settings.job_expire_days)
+    async with MysqlSession() as mysql:
+        result = await mysql.execute(
+            text(f"""
+                UPDATE bg_job
+                SET status = 2, update_time = NOW()
+                WHERE status = 0
+                  AND create_time < DATE_SUB(NOW(), INTERVAL {days} DAY)
+            """),
+        )
+        await mysql.commit()
+        affected = result.rowcount
+
+    if affected > 0:
+        log.info("岗位下架：{}条岗位已标记为失效（超过{}天）", affected, days)

app/services/zombie_recover_service.py

@@ -0,0 +1,42 @@
+"""僵尸恢复服务"""
+
+from sqlalchemy import text
+
+from app.core.database import PgSession, MysqlSession
+from app.core.logger import log
+
+
+async def recover_job_zombie() -> None:
+    """岗位清洗僵尸恢复：超时10分钟的 cleaning → pending"""
+    async with PgSession() as pg:
+        result = await pg.execute(
+            text("""
+                UPDATE app_job_data
+                SET clean_status = 'pending', clean_started_at = NULL
+                WHERE clean_status = 'cleaning'
+                  AND clean_started_at < NOW() - INTERVAL '10 minutes'
+            """)
+        )
+        await pg.commit()
+        affected = result.rowcount
+
+    if affected > 0:
+        log.info("岗位僵尸恢复：重置{}条数据", affected)
+
+
+async def recover_company_zombie() -> None:
+    """公司补充僵尸恢复：超时10分钟的 status=3 → 0"""
+    async with MysqlSession() as mysql:
+        result = await mysql.execute(
+            text("""
+                UPDATE bg_company
+                SET status = 0, update_time = NOW()
+                WHERE status = 3
+                  AND update_time < NOW() - INTERVAL 10 MINUTE
+            """)
+        )
+        await mysql.commit()
+        affected = result.rowcount
+
+    if affected > 0:
+        log.info("公司僵尸恢复：重置{}条数据", affected)