From fc74c9f2b0df966fd4dbd75a040fcd86ba1e1658 Mon Sep 17 00:00:00 2001 From: zk Date: Fri, 24 Jul 2026 16:27:50 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=B5=8F=E8=A7=88=E5=99=A8?= =?UTF-8?q?=E8=83=BD=E5=8A=9B=E5=B0=81=E8=A3=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- requirements.txt | 1 + tool/browser.py | 254 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 255 insertions(+) create mode 100644 tool/browser.py diff --git a/requirements.txt b/requirements.txt index 4b44ec2..973fd52 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ cryptography>=41 rapidocr>=3.9 onnxruntime>=1.17 opencv-python>=4.5 +playwright>=1.49 diff --git a/tool/browser.py b/tool/browser.py new file mode 100644 index 0000000..f80d51a --- /dev/null +++ b/tool/browser.py @@ -0,0 +1,254 @@ +"""Playwright 页面抓取工具。 + +对外仅暴露两个动作： +- open_page(url, wait_ms): 打开页面，返回 page_id +- query(page_id, selector): 选中节点并提取文字、图片 URL 与属性 + +实现上内部常驻一个 Browser；每次 open_page 创建独立 BrowserContext 和 Page， +query 完成后自动释放对应资源。 +""" + +from __future__ import annotations + +import asyncio +import threading +import uuid +from dataclasses import dataclass + +from playwright.async_api import Browser, BrowserContext, Page, async_playwright + + +@dataclass +class NodeSnapshot: + """页面节点快照。""" + + tag: str + text: str + image_urls: list[str] + attrs: dict[str, str] + + +@dataclass +class _PageSession: + context: BrowserContext + page: Page + + +class _BrowserRuntime: + """在后台事件循环中管理 Playwright 生命周期。""" + + def __init__(self) -> None: + self._thread: threading.Thread | None = None + self._loop: asyncio.AbstractEventLoop | None = None + self._ready = threading.Event() + self._lock = threading.RLock() + self._playwright = None + self._browser: Browser | None = None + self._pages: dict[str, _PageSession] = {} + + def _thread_main(self) -> None: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + self._loop = loop + self._ready.set() + loop.run_forever() + + pending = asyncio.all_tasks(loop) + for task in pending: + task.cancel() + + if pending: + loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True)) + + if self._browser is not None: + loop.run_until_complete(self._browser.close()) + self._browser = None + + if self._playwright is not None: + loop.run_until_complete(self._playwright.stop()) + self._playwright = None + + loop.run_until_complete(loop.shutdown_asyncgens()) + loop.close() + + def _ensure_thread(self) -> None: + with self._lock: + if self._thread and self._thread.is_alive(): + return + + self._thread = threading.Thread( + target=self._thread_main, + name="offerpie-playwright", + daemon=True, + ) + self._thread.start() + self._ready.wait() + + def _run(self, coro): + self._ensure_thread() + assert self._loop is not None + future = asyncio.run_coroutine_threadsafe(coro, self._loop) + return future.result() + + async def _ensure_browser(self) -> Browser: + if self._browser is not None: + return self._browser + + if self._playwright is None: + self._playwright = await async_playwright().start() + + self._browser = await self._playwright.chromium.launch(headless=True) + return self._browser + + async def _open_page(self, url: str, wait_ms: int) -> str: + browser = await self._ensure_browser() + context = await browser.new_context() + page = await context.new_page() + await page.goto(url, wait_until="domcontentloaded", timeout=30000) + if wait_ms > 0: + await page.wait_for_timeout(wait_ms) + + page_id = uuid.uuid4().hex + self._pages[page_id] = _PageSession(context=context, page=page) + return page_id + + async def _query(self, page_id: str, selector: str) -> list[NodeSnapshot]: + session = self._pages.get(page_id) + if session is None: + raise KeyError(f"Unknown page_id: {page_id}") + + try: + locator = session.page.locator(selector) + payload = await locator.evaluate_all( + r"""(elements) => elements.map((el) => { + const attrs = {}; + for (const attr of Array.from(el.attributes || [])) { + attrs[attr.name] = attr.value; + } + + const imageUrls = []; + const addImageUrl = (url) => { + if (!url) return; + try { + const absUrl = new URL(url, location.href).href; + if ( + absUrl.startsWith('data:') || + absUrl.startsWith('blob:') || + absUrl === 'about:blank' + ) { + return; + } + imageUrls.push(absUrl); + } catch { + if ( + url.startsWith('data:') || + url.startsWith('blob:') || + url === 'about:blank' + ) { + return; + } + imageUrls.push(url); + } + }; + + const resolveImageUrl = (img) => { + const candidates = [ + img.getAttribute('data-src'), + img.getAttribute('data-original'), + img.getAttribute('data-croporisrc'), + img.currentSrc, + img.src, + img.getAttribute('src'), + ]; + + const srcset = img.getAttribute('srcset'); + if (srcset) { + for (const part of srcset.split(',')) { + const candidate = part.trim().split(/\s+/)[0]; + candidates.push(candidate); + } + } + + for (const candidate of candidates) { + if (!candidate) continue; + if ( + candidate.startsWith('data:') || + candidate.startsWith('blob:') || + candidate === 'about:blank' + ) { + continue; + } + try { + const absUrl = new URL(candidate, location.href).href; + if ( + absUrl.startsWith('data:') || + absUrl.startsWith('blob:') || + absUrl === 'about:blank' + ) { + continue; + } + return absUrl; + } catch { + return candidate; + } + } + + return null; + }; + + if (el.tagName === 'IMG') { + addImageUrl(resolveImageUrl(el)); + } + + for (const img of Array.from(el.querySelectorAll('img'))) { + addImageUrl(resolveImageUrl(img)); + } + + const text = (el.innerText || el.textContent || '').trim(); + + return { + tag: (el.tagName || '').toLowerCase(), + text, + image_urls: Array.from(new Set(imageUrls)), + attrs, + }; + })""" + ) + return [ + NodeSnapshot( + tag=item.get("tag", ""), + text=item.get("text", ""), + image_urls=list(item.get("image_urls", [])), + attrs=dict(item.get("attrs", {})), + ) + for item in payload + ] + finally: + await session.page.close() + await session.context.close() + self._pages.pop(page_id, None) + + +_runtime = _BrowserRuntime() + + +def open_page(url: str, wait_ms: int = 3000) -> str: + """打开页面并返回 page_id。""" + return _runtime._run(_runtime._open_page(url, wait_ms)) + + +def query(page_id: str, selector: str) -> list[NodeSnapshot]: + """按选择器抓取节点，提取文本、图片 URL 与属性。""" + items = _runtime._run(_runtime._query(page_id, selector)) + seen_urls: set[str] = set() + + for item in items: + unique_urls: list[str] = [] + for url in item.image_urls: + if url in seen_urls: + continue + seen_urls.add(url) + unique_urls.append(url) + item.image_urls = unique_urls + + return items