添加浏览器能力封装

requirements.txt

@@ -2,3 +2,4 @@ cryptography>=41
 rapidocr>=3.9
 onnxruntime>=1.17
 opencv-python>=4.5
+playwright>=1.49

tool/browser.py

@@ -0,0 +1,254 @@
+"""Playwright 页面抓取工具。
+
+对外仅暴露两个动作：
+- open_page(url, wait_ms): 打开页面，返回 page_id
+- query(page_id, selector): 选中节点并提取文字、图片 URL 与属性
+
+实现上内部常驻一个 Browser；每次 open_page 创建独立 BrowserContext 和 Page，
+query 完成后自动释放对应资源。
+"""
+
+from __future__ import annotations
+
+import asyncio
+import threading
+import uuid
+from dataclasses import dataclass
+
+from playwright.async_api import Browser, BrowserContext, Page, async_playwright
+
+
+@dataclass
+class NodeSnapshot:
+    """页面节点快照。"""
+
+    tag: str
+    text: str
+    image_urls: list[str]
+    attrs: dict[str, str]
+
+
+@dataclass
+class _PageSession:
+    context: BrowserContext
+    page: Page
+
+
+class _BrowserRuntime:
+    """在后台事件循环中管理 Playwright 生命周期。"""
+
+    def __init__(self) -> None:
+        self._thread: threading.Thread | None = None
+        self._loop: asyncio.AbstractEventLoop | None = None
+        self._ready = threading.Event()
+        self._lock = threading.RLock()
+        self._playwright = None
+        self._browser: Browser | None = None
+        self._pages: dict[str, _PageSession] = {}
+
+    def _thread_main(self) -> None:
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        self._loop = loop
+        self._ready.set()
+        loop.run_forever()
+
+        pending = asyncio.all_tasks(loop)
+        for task in pending:
+            task.cancel()
+
+        if pending:
+            loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True))
+
+        if self._browser is not None:
+            loop.run_until_complete(self._browser.close())
+            self._browser = None
+
+        if self._playwright is not None:
+            loop.run_until_complete(self._playwright.stop())
+            self._playwright = None
+
+        loop.run_until_complete(loop.shutdown_asyncgens())
+        loop.close()
+
+    def _ensure_thread(self) -> None:
+        with self._lock:
+            if self._thread and self._thread.is_alive():
+                return
+
+            self._thread = threading.Thread(
+                target=self._thread_main,
+                name="offerpie-playwright",
+                daemon=True,
+            )
+            self._thread.start()
+            self._ready.wait()
+
+    def _run(self, coro):
+        self._ensure_thread()
+        assert self._loop is not None
+        future = asyncio.run_coroutine_threadsafe(coro, self._loop)
+        return future.result()
+
+    async def _ensure_browser(self) -> Browser:
+        if self._browser is not None:
+            return self._browser
+
+        if self._playwright is None:
+            self._playwright = await async_playwright().start()
+
+        self._browser = await self._playwright.chromium.launch(headless=True)
+        return self._browser
+
+    async def _open_page(self, url: str, wait_ms: int) -> str:
+        browser = await self._ensure_browser()
+        context = await browser.new_context()
+        page = await context.new_page()
+        await page.goto(url, wait_until="domcontentloaded", timeout=30000)
+        if wait_ms > 0:
+            await page.wait_for_timeout(wait_ms)
+
+        page_id = uuid.uuid4().hex
+        self._pages[page_id] = _PageSession(context=context, page=page)
+        return page_id
+
+    async def _query(self, page_id: str, selector: str) -> list[NodeSnapshot]:
+        session = self._pages.get(page_id)
+        if session is None:
+            raise KeyError(f"Unknown page_id: {page_id}")
+
+        try:
+            locator = session.page.locator(selector)
+            payload = await locator.evaluate_all(
+                r"""(elements) => elements.map((el) => {
+                    const attrs = {};
+                    for (const attr of Array.from(el.attributes || [])) {
+                        attrs[attr.name] = attr.value;
+                    }
+
+                    const imageUrls = [];
+                    const addImageUrl = (url) => {
+                        if (!url) return;
+                        try {
+                            const absUrl = new URL(url, location.href).href;
+                            if (
+                                absUrl.startsWith('data:') ||
+                                absUrl.startsWith('blob:') ||
+                                absUrl === 'about:blank'
+                            ) {
+                                return;
+                            }
+                            imageUrls.push(absUrl);
+                        } catch {
+                            if (
+                                url.startsWith('data:') ||
+                                url.startsWith('blob:') ||
+                                url === 'about:blank'
+                            ) {
+                                return;
+                            }
+                            imageUrls.push(url);
+                        }
+                    };
+
+                    const resolveImageUrl = (img) => {
+                        const candidates = [
+                            img.getAttribute('data-src'),
+                            img.getAttribute('data-original'),
+                            img.getAttribute('data-croporisrc'),
+                            img.currentSrc,
+                            img.src,
+                            img.getAttribute('src'),
+                        ];
+
+                        const srcset = img.getAttribute('srcset');
+                        if (srcset) {
+                            for (const part of srcset.split(',')) {
+                                const candidate = part.trim().split(/\s+/)[0];
+                                candidates.push(candidate);
+                            }
+                        }
+
+                        for (const candidate of candidates) {
+                            if (!candidate) continue;
+                            if (
+                                candidate.startsWith('data:') ||
+                                candidate.startsWith('blob:') ||
+                                candidate === 'about:blank'
+                            ) {
+                                continue;
+                            }
+                            try {
+                                const absUrl = new URL(candidate, location.href).href;
+                                if (
+                                    absUrl.startsWith('data:') ||
+                                    absUrl.startsWith('blob:') ||
+                                    absUrl === 'about:blank'
+                                ) {
+                                    continue;
+                                }
+                                return absUrl;
+                            } catch {
+                                return candidate;
+                            }
+                        }
+
+                        return null;
+                    };
+
+                    if (el.tagName === 'IMG') {
+                        addImageUrl(resolveImageUrl(el));
+                    }
+
+                    for (const img of Array.from(el.querySelectorAll('img'))) {
+                        addImageUrl(resolveImageUrl(img));
+                    }
+
+                    const text = (el.innerText || el.textContent || '').trim();
+
+                    return {
+                        tag: (el.tagName || '').toLowerCase(),
+                        text,
+                        image_urls: Array.from(new Set(imageUrls)),
+                        attrs,
+                    };
+                })"""
+            )
+            return [
+                NodeSnapshot(
+                    tag=item.get("tag", ""),
+                    text=item.get("text", ""),
+                    image_urls=list(item.get("image_urls", [])),
+                    attrs=dict(item.get("attrs", {})),
+                )
+                for item in payload
+            ]
+        finally:
+            await session.page.close()
+            await session.context.close()
+            self._pages.pop(page_id, None)
+
+
+_runtime = _BrowserRuntime()
+
+
+def open_page(url: str, wait_ms: int = 3000) -> str:
+    """打开页面并返回 page_id。"""
+    return _runtime._run(_runtime._open_page(url, wait_ms))
+
+
+def query(page_id: str, selector: str) -> list[NodeSnapshot]:
+    """按选择器抓取节点，提取文本、图片 URL 与属性。"""
+    items = _runtime._run(_runtime._query(page_id, selector))
+    seen_urls: set[str] = set()
+
+    for item in items:
+        unique_urls: list[str] = []
+        for url in item.image_urls:
+            if url in seen_urls:
+                continue
+            seen_urls.add(url)
+            unique_urls.append(url)
+        item.image_urls = unique_urls
+
+    return items