岗位清洗，差岗位技能提取

manager/src/main/java/org/jiayunet/mapper/AppJobDataMapper.java

@@ -0,0 +1,13 @@
+package org.jiayunet.mapper;
+
+import org.apache.ibatis.annotations.Mapper;
+import org.jiayunet.pojo.po.AppJobData;
+
+/**
+ * 爬虫岗位原始数据Mapper
+ *
+ * @author zk
+ */
+@Mapper
+public interface AppJobDataMapper extends CommonMapper<AppJobData> {
+}

manager/src/main/java/org/jiayunet/pojo/po/AppJobData.java

@@ -0,0 +1,79 @@
+package org.jiayunet.pojo.po;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+import java.time.Instant;
+
+/**
+ * 爬虫岗位原始数据表（app_job_data）
+ * <p>存储爬虫抓取的原始岗位数据，供清洗服务读取并写入业务表</p>
+ *
+ * @author zk
+ */
+@Data
+@TableName(value = "app_job_data")
+public class AppJobData {
+
+    @TableId(type = IdType.AUTO)
+    private Long id;
+
+    /** 关联爬取任务ID */
+    private Long taskCrawlId;
+
+    /** 职位名称 */
+    private String jobTitle;
+
+    /** 薪资 */
+    private String salary;
+
+    /** 工作地点 */
+    private String location;
+
+    /** 公司名称 */
+    private String company;
+
+    /** 经验要求 */
+    private String experience;
+
+    /** 学历要求 */
+    private String education;
+
+    /** 岗位详情(职责+要求+介绍) */
+    private String description;
+
+    /** 详情页URL */
+    private String detailUrl;
+
+    /** 内容哈希值，用于查重 */
+    private String contentHash;
+
+    /** 数据来源 0=官网 1=平台 */
+    private Integer sources;
+
+    /** 是否独立URL 0=页内展示 1=独立页面 */
+    private Integer isIndependentUrl;
+
+    /** 是否有效 0=无效 1=有效 */
+    private Integer isValid;
+
+    /** 有效期 */
+    private Instant expireAt;
+
+    /** 验证状态 pending=待验证 checking=验证中 checked=已验证 */
+    private String checkStatus;
+
+    /** 清洗状态 0=待清洗 1=清洗中 2=已入库 3=已丢弃 */
+    private Integer cleanStatus;
+
+    /** 上次验证时间 */
+    private Instant lastCheckAt;
+
+    /** 创建时间 */
+    private Instant createdAt;
+
+    /** 更新时间 */
+    private Instant updatedAt;
+}

manager/src/main/java/org/jiayunet/pojo/po/Company.java

@@ -67,7 +67,7 @@ public class Company {
     /** 新闻动态（JSON数组） */
     private String news;
 
-    /** 状态 0=待完善 1=已完善 2=禁用 */
+    /** 状态 0=待完善 1=已完善 2=禁用 3=补充中 */
     private Integer status;
 
     /** 创建时间 */

manager/src/main/java/org/jiayunet/service/DictCacheService.java

@@ -0,0 +1,124 @@
+package org.jiayunet.service;
+
+import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
+import lombok.extern.slf4j.Slf4j;
+import org.jiayunet.mapper.ChinaRegionsCodeMapper;
+import org.jiayunet.mapper.IndustryMapper;
+import org.jiayunet.mapper.JobCategoryMapper;
+import org.jiayunet.pojo.po.ChinaRegionsCode;
+import org.jiayunet.pojo.po.Industry;
+import org.jiayunet.pojo.po.JobCategory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import javax.annotation.PostConstruct;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * 字典数据缓存服务
+ * <p>启动时加载岗位分类、行业、地区数据到内存，供清洗/推荐等业务使用</p>
+ * <p>依赖：JobCategoryMapper、IndustryMapper、ChinaRegionsCodeMapper</p>
+ * <p>使用表：bg_job_category（全量缓存）、bg_industry（全量缓存）、bg_china_regions_code（市级缓存）</p>
+ *
+ * @author zk
+ */
+@Slf4j
+@Service
+public class DictCacheService {
+
+    @Autowired
+    private JobCategoryMapper jobCategoryMapper;
+
+    @Autowired
+    private IndustryMapper industryMapper;
+
+    @Autowired
+    private ChinaRegionsCodeMapper chinaRegionsCodeMapper;
+
+    private List<JobCategory> jobCategoryList;
+    private List<Industry> industryList;
+    private List<ChinaRegionsCode> regionList;
+
+    /** 岗位分类文本（叶子节点，带父级路径），供 AI prompt 使用 */
+    private String jobCategoryText;
+    /** 行业文本（叶子节点，带父级路径），供 AI prompt 使用 */
+    private String industryText;
+
+    /**
+     * 启动时加载全量字典数据
+     * <p>分类/行业全量加载用于构建父级路径，文本只取叶子节点</p>
+     */
+    @PostConstruct
+    public void refresh() {
+        log.info("开始加载字典缓存...");
+
+        jobCategoryList = jobCategoryMapper.selectList(null);
+        industryList = industryMapper.selectList(null);
+
+        // 只缓存省级+市级地区（provinceCode 为 null 是省，provinceCode 不为 null 且 cityCode 为 null 是市）
+        regionList = chinaRegionsCodeMapper.selectList(
+                new LambdaQueryWrapper<ChinaRegionsCode>()
+                        .isNull(ChinaRegionsCode::getCityCode)
+        );
+
+        // 构建岗位分类文本：只取三级（叶子），格式 id:name(一级/二级)
+        Map<Long, String> categoryNameMap = jobCategoryList.stream()
+                .collect(Collectors.toMap(JobCategory::getId, JobCategory::getName));
+
+        jobCategoryText = jobCategoryList.stream()
+                .filter(c -> c.getLevel() == 3)
+                .map(c -> {
+                    String parentName = categoryNameMap.getOrDefault(c.getParentId(), "");
+                    String rootName = categoryNameMap.getOrDefault(c.getRootId(), "");
+                    return c.getId() + ":" + c.getName() + "(" + rootName + "/" + parentName + ")";
+                })
+                .collect(Collectors.joining(", "));
+
+        // 构建行业文本：只取二级（叶子），格式 id:name(一级)
+        Map<Long, String> industryNameMap = industryList.stream()
+                .collect(Collectors.toMap(Industry::getId, Industry::getName));
+        industryText = industryList.stream()
+                .filter(i -> i.getLevel() == 2)
+                .map(i -> {
+                    String parentName = industryNameMap.getOrDefault(i.getParentId(), "");
+                    return i.getId() + ":" + i.getName() + "(" + parentName + ")";
+                })
+                .collect(Collectors.joining(", "));
+
+        long categoryLeafCount = jobCategoryList.stream().filter(c -> c.getLevel() == 3).count();
+        long industryLeafCount = industryList.stream().filter(i -> i.getLevel() == 2).count();
+        log.info("字典缓存加载完成: 岗位分类{}条(叶子{}条), 行业{}条(叶子{}条), 地区{}条",
+                jobCategoryList.size(), categoryLeafCount, industryList.size(), industryLeafCount, regionList.size());
+    }
+
+    /** 获取岗位分类文本（叶子节点，带父级路径，逗号分隔） */
+    public String getJobCategoryText() {
+        return jobCategoryText;
+    }
+
+    /** 获取行业文本（叶子节点，带父级路径，逗号分隔） */
+    public String getIndustryText() {
+        return industryText;
+    }
+
+    /**
+     * 根据城市名匹配地区编码
+     * <p>模糊匹配，如"北京"匹配"北京市"</p>
+     *
+     * @param cityName 城市名
+     * @return region_code，匹配不上返回 null
+     */
+    public String matchRegionCode(String cityName) {
+        if (cityName == null || cityName.isBlank()) {
+            return null;
+        }
+        String name = cityName.replace("市", "").replace("省", "").trim();
+        return regionList.stream()
+                .filter(r -> r.getName().contains(name) || name.contains(r.getName().replace("市", "").replace("省", "")))
+                .map(ChinaRegionsCode::getCode)
+                .findFirst()
+                .orElse(null);
+    }
+}

manager/src/main/java/org/jiayunet/service/JobCleanService.java

@@ -0,0 +1,248 @@
+package org.jiayunet.service;
+
+import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
+import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
+import com.fasterxml.jackson.databind.JsonNode;
+import lombok.extern.slf4j.Slf4j;
+import org.jiayunet.ai.AiChatAbility;
+import org.jiayunet.mapper.AppJobDataMapper;
+import org.jiayunet.mapper.JobMapper;
+import org.jiayunet.pojo.po.AppJobData;
+import org.jiayunet.pojo.po.Job;
+import org.jiayunet.tool.HttpTool;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Service;
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+/**
+ * 岗位清洗服务
+ * <p>定时从 app_job_data 捞取待清洗数据，调用 AI 清洗后写入业务表</p>
+ * <p>依赖：AiChatAbility（AI调用）、DictCacheService（字典缓存）、JobCleanTransactionService（事务操作）</p>
+ * <p>使用表：app_job_data（读取/更新状态）、bg_job（去重查询）</p>
+ *
+ * @author zk
+ */
+@Slf4j
+@Service
+public class JobCleanService {
+
+    @Autowired
+    private AiChatAbility aiChatAbility;
+
+    @Autowired
+    private DictCacheService dictCacheService;
+
+    @Autowired
+    private JobCleanTransactionService jobCleanTransactionService;
+
+    @Autowired
+    private AppJobDataMapper appJobDataMapper;
+
+    @Autowired
+    private JobMapper jobMapper;
+
+    @Value("${app.job-clean.batch-size:20}")
+    private int batchSize;
+
+    @Value("${app.job-clean.thread-pool-size:5}")
+    private int threadPoolSize;
+
+    private ExecutorService executorService;
+
+    /** 初始化线程池 */
+    @javax.annotation.PostConstruct
+    public void init() {
+        executorService = Executors.newFixedThreadPool(threadPoolSize);
+    }
+
+    /**
+     * 定时任务A：岗位清洗（每5分钟）
+     * <p>1. 批量锁定待清洗数据 2. 多线程并发调用AI清洗 3. 写入业务表</p>
+     */
+    @Scheduled(cron = "0 */1 * * * ?")
+    public void cleanJob() {
+        // 批量锁定：原子操作，clean_status 0→1
+        int locked = appJobDataMapper.update(null,
+                new LambdaUpdateWrapper<AppJobData>()
+                        .set(AppJobData::getCleanStatus, 1)
+                        .eq(AppJobData::getCleanStatus, 0)
+                        .eq(AppJobData::getIsValid, 1)
+                        .last("LIMIT " + batchSize));
+
+        if (locked == 0) {
+            return;
+        }
+        log.info("岗位清洗：锁定{}条数据", locked);
+
+        // 查出刚锁定的数据
+        List<AppJobData> dataList = appJobDataMapper.selectList(
+                new LambdaQueryWrapper<AppJobData>()
+                        .eq(AppJobData::getCleanStatus, 1)
+                        .eq(AppJobData::getIsValid, 1)
+                        .last("LIMIT " + batchSize));
+
+        // 多线程并发处理
+        for (AppJobData data : dataList) {
+            executorService.submit(() -> {
+                try {
+                    cleanOne(data);
+                } catch (Exception e) {
+                    log.error("岗位清洗异常, id={}", data.getId(), e);
+                    // 异常时保持 clean_status=1，由僵尸恢复任务重置
+                }
+            });
+        }
+    }
+
+    /**
+     * 定时任务B：僵尸恢复（每30分钟）
+     * <p>将超时10分钟仍在清洗中的数据重置为待清洗</p>
+     */
+    @Scheduled(cron = "0 */30 * * * ?")
+    public void recoverZombie() {
+        int recovered = appJobDataMapper.update(null,
+                new LambdaUpdateWrapper<AppJobData>()
+                        .set(AppJobData::getCleanStatus, 0)
+                        .eq(AppJobData::getCleanStatus, 1)
+                        .lt(AppJobData::getUpdatedAt, Instant.now().minusSeconds(600)));
+
+        if (recovered > 0) {
+            log.info("僵尸恢复：重置{}条数据", recovered);
+        }
+    }
+
+    /**
+     * 清洗单条岗位数据
+     * <p>1. 前置校验 2. 拼prompt调AI 3. 解析结果 4. 写入业务表</p>
+     */
+    public void cleanOne(AppJobData data) {
+        // 1. 前置校验
+        if (data.getDescription() == null || data.getDescription().length() < 20) {
+            jobCleanTransactionService.updateCleanStatus(data.getId(), 3);
+            return;
+        }
+
+        // 2. 拼 prompt
+        String systemPrompt = buildSystemPrompt();
+        String userMessage = buildUserMessage(data);
+
+        // 3. 调用 AI
+        String aiResponse = aiChatAbility.chat(systemPrompt, userMessage);
+
+        // 4. 解析 JSON
+        try {
+            // 去掉可能的 markdown 代码块标记
+            String json = aiResponse.trim();
+            if (json.startsWith("```")) {
+                json = json.replaceAll("^```\\w*\\n?", "").replaceAll("\\n?```$", "").trim();
+            }
+
+            JsonNode root = HttpTool.objectMapper.readTree(json);
+
+            // valid 校验
+            if (!root.path("valid").asBoolean(false)) {
+                jobCleanTransactionService.updateCleanStatus(data.getId(), 3);
+                return;
+            }
+
+            // 5. 去重检查
+            String sourceId = String.valueOf(data.getId());
+            Long existJob = jobMapper.selectCount(
+                    new LambdaQueryWrapper<Job>().eq(Job::getSourceId, sourceId));
+            if (existJob > 0) {
+                jobCleanTransactionService.updateCleanStatus(data.getId(), 2);
+                return;
+            }
+
+            // 6. 公司处理（加锁防并发重复）
+            String companyShortName = root.path("companyShortName").asText("");
+            if (companyShortName.isBlank()) {
+                companyShortName = data.getCompany();
+            }
+            Long companyId = jobCleanTransactionService.findOrCreateCompany(companyShortName);
+
+            // 7. 地区处理
+            List<String> regionCodes = new ArrayList<>();
+            JsonNode citiesNode = root.path("cities");
+            if (citiesNode.isArray()) {
+                for (JsonNode city : citiesNode) {
+                    String code = dictCacheService.matchRegionCode(city.asText());
+                    if (code != null) {
+                        regionCodes.add(code);
+                    }
+                }
+            }
+
+            // 8. 写入业务表（短事务，通过独立Service保证@Transactional生效）
+            jobCleanTransactionService.saveJobData(root, data, companyId, sourceId, regionCodes);
+
+        } catch (Exception e) {
+            log.error("AI 返回解析失败, id={}, response={}", data.getId(), aiResponse, e);
+            // 保持 clean_status=1，由僵尸恢复任务重置
+        }
+    }
+
+    /** 构建系统提示词 */
+    private String buildSystemPrompt() {
+        return """
+                你是一个岗位数据清洗助手。请根据提供的原始岗位数据，提取并结构化为JSON格式。
+                
+                返回JSON格式要求：
+                {
+                  "valid": true/false,
+                  "title": "岗位名称",
+                  "salary": "标准化薪资，如10-20K、面议，无效则null",
+                  "education": 0-4的数字(0=不限 1=大专 2=本科 3=硕士 4=博士),
+                  "minExperience": 最低工作年限数字(不要求则0),
+                  "employmentType": 0或1(0=全职 1=兼职，默认0),
+                  "categoryId": 岗位分类ID(必选，从分类列表中选最接近的),
+                  "industryId": 行业ID(仅当明确提到行业经验要求时设置，列表中无完全匹配则选最相似的，未提到则null),
+                  "description": "岗位职责，保持原文风格，格式化展示",
+                  "requirement": "任职要求，保持原文风格，格式化展示",
+                  "bonus": "加分项，无则null",
+                  "tags": ["核心职能标签，最多5个，如数据分析、产品策略"],
+                  "skillTags": ["技能关键词，最多8个，如Java、Spring Boot"],
+                  "companyShortName": "简洁的公司简称，如字节跳动、中国平安",
+                  "cities": ["工作城市列表，精确到市"]
+                }
+                
+                规则：
+                1. description/requirement/bonus 均从原始的 description+experience+education 内容中提取，保持原文风格
+                2. 岗位标题不存在时，从描述中归纳生成
+                3. 薪资标准化为 10-20K、20K、面议 等格式，无效或空则null
+                4. categoryId 必须从分类列表中选一个，不允许为null
+                5. industryId 仅当描述中明确提到行业经验要求时设置
+                6. tags 是核心职能标签（如数据分析、团队协作），最多5个
+                7. skillTags 是技能关键词（如Java、MySQL），最多8个
+                8. companyShortName 去掉地区后缀、招聘后缀、括号内容，保持简洁
+                9. 只返回JSON，不要其他内容
+                """;
+    }
+
+    /** 构建用户消息 */
+    private String buildUserMessage(AppJobData data) {
+        StringBuilder sb = new StringBuilder();
+        sb.append("【原始数据】\n");
+        sb.append("岗位名称: ").append(nullToEmpty(data.getJobTitle())).append("\n");
+        sb.append("薪资: ").append(nullToEmpty(data.getSalary())).append("\n");
+        sb.append("工作地点: ").append(nullToEmpty(data.getLocation())).append("\n");
+        sb.append("公司: ").append(nullToEmpty(data.getCompany())).append("\n");
+        sb.append("经验要求: ").append(nullToEmpty(data.getExperience())).append("\n");
+        sb.append("学历要求: ").append(nullToEmpty(data.getEducation())).append("\n");
+        sb.append("岗位详情: ").append(nullToEmpty(data.getDescription())).append("\n\n");
+        sb.append("【岗位分类列表】\n").append(dictCacheService.getJobCategoryText()).append("\n\n");
+        sb.append("【行业列表】\n").append(dictCacheService.getIndustryText());
+        return sb.toString();
+    }
+
+    private String nullToEmpty(String s) {
+        return s == null ? "" : s;
+    }
+}

manager/src/main/java/org/jiayunet/service/JobCleanTransactionService.java

@@ -0,0 +1,123 @@
+package org.jiayunet.service;
+
+import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
+import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
+import com.fasterxml.jackson.databind.JsonNode;
+import lombok.extern.slf4j.Slf4j;
+import org.jiayunet.mapper.AppJobDataMapper;
+import org.jiayunet.mapper.CompanyMapper;
+import org.jiayunet.mapper.JobMapper;
+import org.jiayunet.mapper.JobRegionRelationMapper;
+import org.jiayunet.pojo.po.AppJobData;
+import org.jiayunet.pojo.po.Company;
+import org.jiayunet.pojo.po.Job;
+import org.jiayunet.pojo.po.JobRegionRelation;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+import java.time.Instant;
+import java.util.List;
+
+/**
+ * 岗位清洗事务服务
+ * <p>独立出来解决 @Transactional 同类自调用失效问题</p>
+ * <p>依赖：JobMapper、CompanyMapper、JobRegionRelationMapper、AppJobDataMapper</p>
+ * <p>使用表：bg_job（写入）、bg_company（查询/创建）、bg_job_region_relation（写入）、app_job_data（更新状态）</p>
+ *
+ * @author zk
+ */
+@Slf4j
+@Service
+public class JobCleanTransactionService {
+
+    @Autowired
+    private JobMapper jobMapper;
+
+    @Autowired
+    private CompanyMapper companyMapper;
+
+    @Autowired
+    private JobRegionRelationMapper jobRegionRelationMapper;
+
+    @Autowired
+    private AppJobDataMapper appJobDataMapper;
+
+    /**
+     * 写入 bg_job + bg_job_region_relation + 更新 clean_status（短事务）
+     */
+    @Transactional(rollbackFor = Exception.class)
+    public void saveJobData(JsonNode root, AppJobData data, Long companyId, String sourceId, List<String> regionCodes) {
+        Job job = new Job();
+        job.setTitle(root.path("title").asText(""));
+        job.setCompanyId(companyId);
+        job.setCategoryId(root.path("categoryId").asLong(0));
+        job.setEmploymentType(root.path("employmentType").asInt(0));
+        job.setDescription(root.path("description").asText(""));
+        job.setRequirement(root.path("requirement").asText(""));
+        job.setBonus(root.path("bonus").asText(null));
+        job.setTags(root.path("tags").toString());
+        job.setSkillTags(root.path("skillTags").toString());
+
+        String salary = root.path("salary").asText(null);
+        job.setSalary("null".equals(salary) ? null : salary);
+
+        job.setEducation(root.path("education").asInt(0));
+        job.setMinExperience(root.path("minExperience").asInt(0));
+
+        Long industryId = root.path("industryId").asLong(0);
+        job.setIndustryId(industryId == 0 ? null : industryId);
+
+        job.setSourceUrl(data.getDetailUrl());
+        job.setSourceId(sourceId);
+        job.setStatus(0);
+        job.setCreateTime(Instant.now());
+        job.setUpdateTime(Instant.now());
+
+        jobMapper.insert(job);
+
+        // 写入岗位-地区关联
+        for (String regionCode : regionCodes) {
+            JobRegionRelation relation = new JobRegionRelation();
+            relation.setJobId(job.getId());
+            relation.setRegionCode(regionCode);
+            relation.setCreateTime(Instant.now());
+            jobRegionRelationMapper.insert(relation);
+        }
+
+        // 更新清洗状态
+        updateCleanStatus(data.getId(), 2);
+    }
+
+    /**
+     * 查找或创建公司（加锁防并发重复）
+     * <p>按 short_name 查询，不存在则创建一条待完善记录</p>
+     */
+    public synchronized Long findOrCreateCompany(String shortName) {
+        Company company = companyMapper.selectOne(
+                new LambdaQueryWrapper<Company>()
+                        .eq(Company::getShortName, shortName)
+                        .last("LIMIT 1"));
+
+        if (company != null) {
+            return company.getId();
+        }
+
+        Company newCompany = new Company();
+        newCompany.setName(shortName);
+        newCompany.setShortName(shortName);
+        newCompany.setStatus(0);
+        newCompany.setCreateTime(Instant.now());
+        newCompany.setUpdateTime(Instant.now());
+        companyMapper.insert(newCompany);
+        return newCompany.getId();
+    }
+
+    /** 更新清洗状态 */
+    public void updateCleanStatus(Long id, int status) {
+        appJobDataMapper.update(null,
+                new LambdaUpdateWrapper<AppJobData>()
+                        .set(AppJobData::getCleanStatus, status)
+                        .eq(AppJobData::getId, id));
+    }
+}