"""Playwright 页面抓取工具。 对外仅暴露两个动作: - open_page(url, wait_ms): 打开页面,返回 page_id - query(page_id, selector): 选中节点并提取文字、图片 URL 与属性 实现上内部常驻一个 Browser;每次 open_page 创建独立 BrowserContext 和 Page。 Playwright 只在一个后台线程里运行,但外层 API 保持同步。 """ from __future__ import annotations import atexit import queue import threading import uuid from concurrent.futures import Future from dataclasses import dataclass from typing import Any, Callable from playwright.sync_api import Browser, BrowserContext, Page, sync_playwright @dataclass class NodeSnapshot: """页面节点快照。""" tag: str text: str image_urls: list[str] attrs: dict[str, str] @dataclass class _PageSession: context: BrowserContext page: Page class _BrowserRuntime: """在单独线程中管理 Playwright 生命周期。""" def __init__(self) -> None: self._thread: threading.Thread | None = None self._ready = threading.Event() self._closed = threading.Event() self._job_queue: queue.Queue[ tuple[Callable[..., Any], tuple[Any, ...], dict[str, Any], Future[Any]] ] = queue.Queue() self._lock = threading.RLock() self._playwright = None self._browser: Browser | None = None self._pages: dict[str, _PageSession] = {} atexit.register(self.close) def _thread_main(self) -> None: with sync_playwright() as playwright: self._playwright = playwright self._browser = playwright.chromium.launch(headless=True) self._ready.set() while True: job = self._job_queue.get() if job is None: break fn, args, kwargs, fut = job try: result = fn(*args, **kwargs) except BaseException as exc: # pragma: no cover - propagate failures fut.set_exception(exc) else: fut.set_result(result) for session in self._pages.values(): try: session.page.close() except Exception: pass try: session.context.close() except Exception: pass self._pages.clear() if self._browser is not None: try: self._browser.close() finally: self._browser = None self._playwright = None self._closed.set() def _ensure_thread(self) -> None: with self._lock: if self._thread and self._thread.is_alive(): return self._thread = threading.Thread( target=self._thread_main, name="offerpie-playwright", daemon=True, ) self._thread.start() self._ready.wait() def _run(self, fn: Callable[..., Any], *args: Any, **kwargs: Any) -> Any: self._ensure_thread() fut: Future[Any] = Future() self._job_queue.put((fn, args, kwargs, fut)) return fut.result() def close(self) -> None: with self._lock: if self._closed.is_set(): return if self._thread is None: return self._job_queue.put(None) self._closed.wait(timeout=10) def _ensure_browser(self) -> Browser: if self._browser is None: raise RuntimeError("Browser runtime is not ready.") return self._browser def _open_page_impl(self, url: str, wait_ms: int) -> str: browser = self._ensure_browser() context = browser.new_context() page = context.new_page() page.goto(url, wait_until="domcontentloaded", timeout=30000) if wait_ms > 0: page.wait_for_timeout(wait_ms) page_id = uuid.uuid4().hex self._pages[page_id] = _PageSession(context=context, page=page) return page_id def _query_impl(self, page_id: str, selector: str) -> list[NodeSnapshot]: session = self._pages.get(page_id) if session is None: raise KeyError(f"Unknown page_id: {page_id}") locator = session.page.locator(selector) payload = locator.evaluate_all( r"""(elements) => elements.map((el) => { const attrs = {}; for (const attr of Array.from(el.attributes || [])) { attrs[attr.name] = attr.value; } const imageUrls = []; const addImageUrl = (url) => { if (!url) return; try { const absUrl = new URL(url, location.href).href; if ( absUrl.startsWith('data:') || absUrl.startsWith('blob:') || absUrl === 'about:blank' ) { return; } imageUrls.push(absUrl); } catch { if ( url.startsWith('data:') || url.startsWith('blob:') || url === 'about:blank' ) { return; } imageUrls.push(url); } }; const resolveImageUrl = (img) => { const candidates = [ img.getAttribute('data-src'), img.getAttribute('data-original'), img.getAttribute('data-croporisrc'), img.currentSrc, img.src, img.getAttribute('src'), ]; const srcset = img.getAttribute('srcset'); if (srcset) { for (const part of srcset.split(',')) { const candidate = part.trim().split(/\s+/)[0]; candidates.push(candidate); } } for (const candidate of candidates) { if (!candidate) continue; if ( candidate.startsWith('data:') || candidate.startsWith('blob:') || candidate === 'about:blank' ) { continue; } try { const absUrl = new URL(candidate, location.href).href; if ( absUrl.startsWith('data:') || absUrl.startsWith('blob:') || absUrl === 'about:blank' ) { continue; } return absUrl; } catch { return candidate; } } return null; }; if (el.tagName === 'IMG') { addImageUrl(resolveImageUrl(el)); } for (const img of Array.from(el.querySelectorAll('img'))) { addImageUrl(resolveImageUrl(img)); } const text = (el.innerText || el.textContent || '').trim(); return { tag: (el.tagName || '').toLowerCase(), text, image_urls: Array.from(new Set(imageUrls)), attrs, }; })""" ) return [ NodeSnapshot( tag=item.get("tag", ""), text=item.get("text", ""), image_urls=list(item.get("image_urls", [])), attrs=dict(item.get("attrs", {})), ) for item in payload ] _runtime = _BrowserRuntime() def open_page(url: str, wait_ms: int = 3000) -> str: """打开页面并返回 page_id。""" return _runtime._run(_runtime._open_page_impl, url, wait_ms) def query(page_id: str, selector: str) -> list[NodeSnapshot]: """按选择器抓取节点,提取文本、图片 URL 与属性。""" items = _runtime._run(_runtime._query_impl, page_id, selector) seen_urls: set[str] = set() for item in items: unique_urls: list[str] = [] for url in item.image_urls: if url in seen_urls: continue seen_urls.add(url) unique_urls.append(url) item.image_urls = unique_urls return items