From 1a57be892138224622bf1389eb6c727f9cfc2654 Mon Sep 17 00:00:00 2001 From: zk Date: Thu, 30 Jul 2026 11:45:08 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E9=83=A8=E7=BD=B2=E6=96=87?= =?UTF-8?q?=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.prod | 2 +- Dockerfile | 50 ++++++++++++++ Jenkinsfile | 165 ++++++++++++++++++++++++++++++++++++++++++++ app/tool/browser.py | 9 ++- docker-compose.yml | 22 ++++++ 5 files changed, 246 insertions(+), 2 deletions(-) create mode 100644 Dockerfile create mode 100644 Jenkinsfile create mode 100644 docker-compose.yml diff --git a/.env.prod b/.env.prod index 022df3c..482fcac 100644 --- a/.env.prod +++ b/.env.prod @@ -30,4 +30,4 @@ OSS_BUCKET=offerpie OSS_DOMAIN=https://offerpie.oss-cn-guangzhou.aliyuncs.com # 公告落库并发线程数（不要超过 MYSQL_POOL_SIZE + MYSQL_MAX_OVERFLOW） -SPIDER_MAX_WORKERS=12 +SPIDER_MAX_WORKERS=16 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..40ece67 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,50 @@ +# 使用 Python 3.12 slim 镜像 +FROM python:3.12-slim + +ENV TZ=Asia/Shanghai +ENV ENV=prod +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 + +# Playwright 浏览器统一装到系统目录，避免落在 HOME 里 +ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright +# 浏览器下载走 npmmirror 镜像，国内构建更快（网络可直连时可去掉） +ENV PLAYWRIGHT_DOWNLOAD_HOST=https://cdn.npmmirror.com/binaries/playwright + +# 时区 + 系统依赖（libgl1、libglib2.0-0 是 opencv-python 的运行时依赖） +RUN ln -sf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone \ + && rm -rf /etc/apt/sources.list.d/* \ + && echo "deb https://mirrors.aliyun.com/debian/ bookworm main non-free contrib" > /etc/apt/sources.list \ + && echo "deb https://mirrors.aliyun.com/debian-security/ bookworm-security main non-free contrib" >> /etc/apt/sources.list \ + && echo "deb https://mirrors.aliyun.com/debian/ bookworm-updates main non-free contrib" >> /etc/apt/sources.list \ + && apt-get clean \ + && apt-get update \ + && apt-get install -y --no-install-recommends curl libgl1 libglib2.0-0 \ + && rm -rf /var/lib/apt/lists/* + +RUN mkdir -p /app/logs + +WORKDIR /app + +# 先拷贝依赖声明，利用 Docker 层缓存 +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt \ + -i https://mirrors.aliyun.com/pypi/simple/ \ + --trusted-host mirrors.aliyun.com + +# 安装 Chromium 及其系统依赖（供 app/tool/browser.py 使用） +RUN playwright install --with-deps chromium \ + && rm -rf /var/lib/apt/lists/* + +# 预热 OCR 模型，避免首次采集时才去下载 onnx 模型 +# 参数与 app/tool/ocr.py 保持一致，确保命中同一份模型缓存 +RUN python -c "from rapidocr import RapidOCR; RapidOCR(params={'Global.use_cls': False, 'Global.text_score': 0.5})" + +# 拷贝应用代码、一次性初始化脚本（含 init_xiaozhaoya 依赖的 doc/ 数据）和环境配置 +COPY app/ ./app/ +COPY doc/ ./doc/ +COPY init_offershow.py init_offerqingbaoju.py init_xiaozhaoya.py ./ +COPY .env.prod ./.env.prod + +# 纯后台定时任务服务，无 HTTP 端口 +CMD ["python", "-m", "app.main"] diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..052d41e --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,165 @@ +/** + * OfferPie Spider 部署流水线 + * + * Jenkins 容器挂载了宿主机 docker.sock，镜像共享。 + * 直接在 workspace 内 build + compose up。 + * + * SCRIPT=main 时部署常驻定时任务服务； + * SCRIPT=init_* 时用同一个镜像跑一次性存量初始化脚本，跑完容器自动删除，不影响常驻服务。 + */ +pipeline { + agent any + + parameters { + choice(name: 'BRANCH', choices: ['master', 'dev'], description: '选择要部署的分支') + choice(name: 'ACTION', choices: ['deploy', 'stop'], description: '操作：deploy=构建部署，stop=停止服务') + choice(name: 'SCRIPT', + choices: ['main', 'init_offershow', 'init_offerqingbaoju', 'init_xiaozhaoya'], + description: '选择要运行的脚本：main=常驻定时任务服务，init_*=一次性存量初始化（跑完退出）') + string(name: 'LIMIT', defaultValue: '', + description: '抓取条数上限，仅对 init_offershow / init_offerqingbaoju 生效；留空用脚本默认值') + } + + environment { + IMAGE_NAME = 'offerpie-spider' + IMAGE_TAG = 'latest' + CONTAINER_NAME = 'offerpie-spider' + // 固定 compose 项目名：默认取 workspace 目录名，并发构建时会变成 xxx@2 导致容器名冲突 + COMPOSE_PROJECT_NAME = 'offerpie-spider' + // 一次性容器的挂载与资源限制，保持与 docker-compose.yml 一致 + LOG_VOLUME = '/opt/offerpie/spider/logs:/app/logs' + } + + stages { + stage('参数校验') { + steps { + script { + // LIMIT 会拼进 shell 命令，只允许纯数字，防止注入 + def limit = params.LIMIT?.trim() + if (limit && !(limit ==~ /[1-9]\d*/)) { + error "LIMIT 必须是正整数，当前值：${params.LIMIT}" + } + if (limit && params.SCRIPT in ['main', 'init_xiaozhaoya']) { + echo "⚠️ ${params.SCRIPT} 不接受 LIMIT 参数，本次将忽略" + } + } + } + } + + stage('停止服务') { + when { + expression { params.ACTION == 'stop' } + } + steps { + sh "docker compose down || true" + } + } + + stage('拉取代码') { + when { + expression { params.ACTION == 'deploy' } + } + steps { + echo "拉取 ${params.BRANCH} 分支代码" + git branch: "${params.BRANCH}", + credentialsId: 'gitea-fab089c1-b55d-4b58-9fad', + url: 'https://git.jianshixingqiu.com/offerpai/campus_spider.git' + } + } + + stage('构建镜像') { + when { + expression { params.ACTION == 'deploy' } + } + steps { + sh "docker build -t ${IMAGE_NAME}:${IMAGE_TAG} ." + } + } + + stage('部署常驻服务') { + when { + expression { params.ACTION == 'deploy' && params.SCRIPT == 'main' } + } + steps { + // 停旧启新，docker-compose.yml 就在当前 workspace + sh "docker compose down || true" + sh "docker compose up -d" + sleep 5 + // 精确匹配容器名，避免把 offerpie-spider-init_* 这类一次性容器也列出来 + sh "docker ps -f 'name=^${CONTAINER_NAME}\$' --format '{{.Status}}'" + } + } + + stage('执行初始化脚本') { + when { + expression { params.ACTION == 'deploy' && params.SCRIPT != 'main' } + } + steps { + script { + def limit = params.LIMIT?.trim() + def runName = "${CONTAINER_NAME}-${params.SCRIPT}" + def cmd = "python ${params.SCRIPT}.py" + if (limit && params.SCRIPT != 'init_xiaozhaoya') { + cmd += " ${limit}" + } + + echo "运行一次性脚本：${cmd}" + + // 同名任务还在跑就直接失败，不要把正在灌数的任务强杀掉 + def running = sh(script: "docker ps -q -f 'name=^${runName}\$'", + returnStdout: true).trim() + if (running) { + error "${runName} 正在运行中，如确认要重跑请先手动执行：docker rm -f ${runName}" + } + + // 清掉已退出的同名残留容器（--rm 一般会自动删，异常退出时可能留下） + sh "docker rm -f ${runName} || true" + // 故意用 docker run 而不是 docker compose run： + // compose run 起的容器带同一个 project 标签，别人执行 compose down 会把灌数任务一起干掉 + // LOG_FILE_NAME 让日志单独落一个文件，不和常驻服务的 spider.log 交叉写入 + // --rm：跑完自动删除；不带 -d，日志直接输出到本次构建，脚本退出码决定构建成败 + // 实测全量灌数约 12 小时，超时给到 24 小时 + timeout(time: 24, unit: 'HOURS') { + sh """docker run --rm --name ${runName} \ + --network host \ + --shm-size 2g \ + --memory 16g \ + --cpus 8 \ + -e ENV=prod \ + -e TZ=Asia/Shanghai \ + -e OMP_NUM_THREADS=8 \ + -e LOG_FILE_NAME=${params.SCRIPT}.log \ + -v ${LOG_VOLUME} \ + ${IMAGE_NAME}:${IMAGE_TAG} ${cmd}""" + } + } + } + } + + stage('清理') { + when { + expression { params.ACTION == 'deploy' } + } + steps { + sh "docker image prune -f || true" + } + } + } + + post { + success { + script { + if (params.ACTION == 'stop') { + echo "✅ 服务已停止" + } else if (params.SCRIPT == 'main') { + echo "✅ 部署成功！容器 ${CONTAINER_NAME} 已启动" + } else { + echo "✅ 初始化脚本 ${params.SCRIPT}.py 执行完成" + } + } + } + failure { + echo '❌ 操作失败，请检查日志' + } + } +} diff --git a/app/tool/browser.py b/app/tool/browser.py index 853e2d5..81815f8 100644 --- a/app/tool/browser.py +++ b/app/tool/browser.py @@ -25,6 +25,11 @@ from playwright.async_api import Browser, BrowserContext, Page, async_playwright # 页面打开超时（毫秒） _GOTO_TIMEOUT = 30000 +# Chromium 启动参数： +# --no-sandbox 容器内以 root 运行时 Chrome 沙箱不可用，不加会直接启动失败 +# --disable-dev-shm-usage 容器 /dev/shm 偏小时改用磁盘，避免渲染进程崩溃 +_LAUNCH_ARGS = ["--no-sandbox", "--disable-dev-shm-usage"] + @dataclass class NodeSnapshot: @@ -202,7 +207,9 @@ class _BrowserRuntime: async with self._launch_lock: if self._browser is None: self._playwright = await async_playwright().start() - self._browser = await self._playwright.chromium.launch(headless=True) + self._browser = await self._playwright.chromium.launch( + headless=True, args=_LAUNCH_ARGS + ) return self._browser async def _open_page_impl(self, url: str, wait_ms: int) -> str: diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..888704f --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,22 @@ +services: + spider: + image: offerpie-spider:latest + container_name: offerpie-spider + restart: unless-stopped + network_mode: host + environment: + - ENV=prod + - TZ=Asia/Shanghai + # OCR 推理线程数，与下面的 cpus 配额保持一致 + - OMP_NUM_THREADS=8 + volumes: + - /opt/offerpie/spider/logs:/app/logs + # Chromium 共享内存，页面渲染走内存比落盘快 + shm_size: 2gb + deploy: + resources: + limits: + # 宿主 32G，实测全量采集峰值约 4G，上限给足避免 OOM kill + memory: 8G + # 宿主 12 核 + cpus: '8'