From d7f1ba481277be37c09d3c8259fd89751962a4af Mon Sep 17 00:00:00 2001 From: zk Date: Fri, 24 Jul 2026 17:55:27 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B7=BB=E5=8A=A0Ai=20=E8=83=BD=E5=8A=9B?= =?UTF-8?q?=EF=BC=8C=E9=87=8D=E6=9E=84=E9=A1=B9=E7=9B=AE=E7=BB=93=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 3 ++ app/__init__.py | 1 + app/ai/__init__.py | 1 + app/ai/model_config.py | 13 +++++++ app/ai/models.py | 64 +++++++++++++++++++++++++++++++++ app/config/__init__.py | 18 ++++++++++ app/config/settings.py | 22 ++++++++++++ app/tool/__init__.py | 1 + {tool => app/tool}/browser.py | 0 {tool => app/tool}/cv.py | 0 {tool => app/tool}/ocr.py | 0 {tool => app/tool}/qr_decode.py | 0 requirements.txt | 4 +++ 13 files changed, 127 insertions(+) create mode 100644 app/__init__.py create mode 100644 app/ai/__init__.py create mode 100644 app/ai/model_config.py create mode 100644 app/ai/models.py create mode 100644 app/config/__init__.py create mode 100644 app/config/settings.py create mode 100644 app/tool/__init__.py rename {tool => app/tool}/browser.py (100%) rename {tool => app/tool}/cv.py (100%) rename {tool => app/tool}/ocr.py (100%) rename {tool => app/tool}/qr_decode.py (100%) diff --git a/.gitignore b/.gitignore index 1dbc335..c3b5d5f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ .venv/ +.env +.env.test +.env.prod .idea/ .vscode/ .history/ diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..b5ad1fa --- /dev/null +++ b/app/__init__.py @@ -0,0 +1 @@ +"""OfferPie 爬虫应用包。""" diff --git a/app/ai/__init__.py b/app/ai/__init__.py new file mode 100644 index 0000000..c9539f4 --- /dev/null +++ b/app/ai/__init__.py @@ -0,0 +1 @@ +"""AI 模型配置。""" diff --git a/app/ai/model_config.py b/app/ai/model_config.py new file mode 100644 index 0000000..f8d4ddb --- /dev/null +++ b/app/ai/model_config.py @@ -0,0 +1,13 @@ +"""AI 模型场景配置。 + +集中管理爬虫服务使用的模型，切换模型只需修改此文件。 +""" + +from app.ai.models import LLM + + +class SpiderModel: + """爬虫 AI 场景模型。""" + + # 页面内容理解与结构化提取 + CONTENT_EXTRACT = LLM.CLAUDE_OPUS.create(temperature=0) diff --git a/app/ai/models.py b/app/ai/models.py new file mode 100644 index 0000000..12457b2 --- /dev/null +++ b/app/ai/models.py @@ -0,0 +1,64 @@ +"""LLM 模型枚举与实例获取。 + +Usage: + from app.ai.models import LLM + + llm = LLM.DOUBAO_SEED_LITE.create(temperature=0) + llm = LLM.CLAUDE_OPUS.create(temperature=0) +""" + +from enum import Enum +from typing import Callable + +from langchain_anthropic import ChatAnthropic +from langchain_core.language_models import BaseChatModel +from langchain_openai import ChatOpenAI + +from app.config import settings + +ConfigGetter = Callable[[], str] + +# 供应商连接配置 = (api_key函数, base_url函数) +_VOLCENGINE = ( + lambda: settings.volcengine_api_key, + lambda: settings.volcengine_base_url, +) +_ANTHROPIC = ( + lambda: settings.anthropic_api_key, + lambda: settings.anthropic_base_url, +) + + +class LLM(Enum): + """所有可用模型，每个枚举值 = (模型名, 封装类, api_key函数, base_url函数)。""" + + # 火山引擎（OpenAI 兼容） + DOUBAO_PRO_32K = ("doubao-1-5-pro-32k-250115", ChatOpenAI, *_VOLCENGINE) + DOUBAO_LITE_32K = ("doubao-1-5-lite-32k-250115", ChatOpenAI, *_VOLCENGINE) + DOUBAO_SEED_LITE = ("doubao-seed-2-0-lite-260215", ChatOpenAI, *_VOLCENGINE) + DOUBAO_SEED_PRO = ("doubao-seed-2-0-pro-260215", ChatOpenAI, *_VOLCENGINE) + DEEPSEEK_V4_FLASH = ("deepseek-v4-flash-260425", ChatOpenAI, *_VOLCENGINE) + + # Claude（Anthropic 风格） + CLAUDE_OPUS = ("claude-opus-4-6", ChatAnthropic, *_ANTHROPIC) + + def __init__( + self, + model_name: str, + model_class: type[BaseChatModel], + api_key_getter: ConfigGetter, + base_url_getter: ConfigGetter, + ) -> None: + self.model_name = model_name + self._model_class = model_class + self._api_key_getter = api_key_getter + self._base_url_getter = base_url_getter + + def create(self, **kwargs) -> BaseChatModel: + """创建模型实例，其他参数透传给 LangChain 模型类。""" + return self._model_class( + model=self.model_name, + api_key=self._api_key_getter(), + base_url=self._base_url_getter(), + **kwargs, + ) diff --git a/app/config/__init__.py b/app/config/__init__.py new file mode 100644 index 0000000..900fc27 --- /dev/null +++ b/app/config/__init__.py @@ -0,0 +1,18 @@ +import os +from pathlib import Path + +from .settings import Settings + +_env = os.getenv("ENV", "dev") +_env_files = {"dev": ".env", "test": ".env.test", "prod": ".env.prod"} + +# 定位项目根目录（config 上两级） +_project_root = Path(__file__).resolve().parent.parent.parent +_env_file = _project_root / _env_files.get(_env, ".env") + +if not _env_file.exists(): + raise FileNotFoundError(f".env 文件不存在: {_env_file}") + +settings = Settings(_env_file=str(_env_file)) + +__all__ = ["settings"] diff --git a/app/config/settings.py b/app/config/settings.py new file mode 100644 index 0000000..6b7271a --- /dev/null +++ b/app/config/settings.py @@ -0,0 +1,22 @@ +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + """项目配置，通过环境变量或项目根目录下的 .env 文件覆盖。""" + + env: str = "dev" + + # 火山引擎（OpenAI 兼容风格） + volcengine_api_key: str = "" + volcengine_base_url: str = "https://ark.cn-beijing.volces.com/api/v3" + + # Claude（Anthropic 风格） + anthropic_api_key: str = "" + anthropic_base_url: str = "" + + model_config = SettingsConfigDict( + env_file=".env", + env_file_encoding="utf-8", + case_sensitive=False, + extra="ignore", + ) diff --git a/app/tool/__init__.py b/app/tool/__init__.py new file mode 100644 index 0000000..7418a44 --- /dev/null +++ b/app/tool/__init__.py @@ -0,0 +1 @@ +"""爬虫工具集。""" diff --git a/tool/browser.py b/app/tool/browser.py similarity index 100% rename from tool/browser.py rename to app/tool/browser.py diff --git a/tool/cv.py b/app/tool/cv.py similarity index 100% rename from tool/cv.py rename to app/tool/cv.py diff --git a/tool/ocr.py b/app/tool/ocr.py similarity index 100% rename from tool/ocr.py rename to app/tool/ocr.py diff --git a/tool/qr_decode.py b/app/tool/qr_decode.py similarity index 100% rename from tool/qr_decode.py rename to app/tool/qr_decode.py diff --git a/requirements.txt b/requirements.txt index 973fd52..c6284f1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,7 @@ rapidocr>=3.9 onnxruntime>=1.17 opencv-python>=4.5 playwright>=1.49 +pydantic-settings>=2.0 +langchain-openai>=0.3 +langchain-anthropic>=0.3 +langchain-core>=0.3