"""Playwright 页面抓取工具。 对外仅暴露两个动作: - open_page(url, wait_ms): 打开页面,返回 page_id - query(page_id, selector): 选中节点并提取文字、图片 URL 与属性 实现上内部常驻一个 Browser;每次 open_page 创建独立 BrowserContext 和 Page, query 完成后自动释放对应资源。 """ from __future__ import annotations import asyncio import threading import uuid from dataclasses import dataclass from playwright.async_api import Browser, BrowserContext, Page, async_playwright @dataclass class NodeSnapshot: """页面节点快照。""" tag: str text: str image_urls: list[str] attrs: dict[str, str] @dataclass class _PageSession: context: BrowserContext page: Page class _BrowserRuntime: """在后台事件循环中管理 Playwright 生命周期。""" def __init__(self) -> None: self._thread: threading.Thread | None = None self._loop: asyncio.AbstractEventLoop | None = None self._ready = threading.Event() self._lock = threading.RLock() self._playwright = None self._browser: Browser | None = None self._pages: dict[str, _PageSession] = {} def _thread_main(self) -> None: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) self._loop = loop self._ready.set() loop.run_forever() pending = asyncio.all_tasks(loop) for task in pending: task.cancel() if pending: loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True)) if self._browser is not None: loop.run_until_complete(self._browser.close()) self._browser = None if self._playwright is not None: loop.run_until_complete(self._playwright.stop()) self._playwright = None loop.run_until_complete(loop.shutdown_asyncgens()) loop.close() def _ensure_thread(self) -> None: with self._lock: if self._thread and self._thread.is_alive(): return self._thread = threading.Thread( target=self._thread_main, name="offerpie-playwright", daemon=True, ) self._thread.start() self._ready.wait() def _run(self, coro): self._ensure_thread() assert self._loop is not None future = asyncio.run_coroutine_threadsafe(coro, self._loop) return future.result() async def _ensure_browser(self) -> Browser: if self._browser is not None: return self._browser if self._playwright is None: self._playwright = await async_playwright().start() self._browser = await self._playwright.chromium.launch(headless=True) return self._browser async def _open_page(self, url: str, wait_ms: int) -> str: browser = await self._ensure_browser() context = await browser.new_context() page = await context.new_page() await page.goto(url, wait_until="domcontentloaded", timeout=30000) if wait_ms > 0: await page.wait_for_timeout(wait_ms) page_id = uuid.uuid4().hex self._pages[page_id] = _PageSession(context=context, page=page) return page_id async def _query(self, page_id: str, selector: str) -> list[NodeSnapshot]: session = self._pages.get(page_id) if session is None: raise KeyError(f"Unknown page_id: {page_id}") try: locator = session.page.locator(selector) payload = await locator.evaluate_all( r"""(elements) => elements.map((el) => { const attrs = {}; for (const attr of Array.from(el.attributes || [])) { attrs[attr.name] = attr.value; } const imageUrls = []; const addImageUrl = (url) => { if (!url) return; try { const absUrl = new URL(url, location.href).href; if ( absUrl.startsWith('data:') || absUrl.startsWith('blob:') || absUrl === 'about:blank' ) { return; } imageUrls.push(absUrl); } catch { if ( url.startsWith('data:') || url.startsWith('blob:') || url === 'about:blank' ) { return; } imageUrls.push(url); } }; const resolveImageUrl = (img) => { const candidates = [ img.getAttribute('data-src'), img.getAttribute('data-original'), img.getAttribute('data-croporisrc'), img.currentSrc, img.src, img.getAttribute('src'), ]; const srcset = img.getAttribute('srcset'); if (srcset) { for (const part of srcset.split(',')) { const candidate = part.trim().split(/\s+/)[0]; candidates.push(candidate); } } for (const candidate of candidates) { if (!candidate) continue; if ( candidate.startsWith('data:') || candidate.startsWith('blob:') || candidate === 'about:blank' ) { continue; } try { const absUrl = new URL(candidate, location.href).href; if ( absUrl.startsWith('data:') || absUrl.startsWith('blob:') || absUrl === 'about:blank' ) { continue; } return absUrl; } catch { return candidate; } } return null; }; if (el.tagName === 'IMG') { addImageUrl(resolveImageUrl(el)); } for (const img of Array.from(el.querySelectorAll('img'))) { addImageUrl(resolveImageUrl(img)); } const text = (el.innerText || el.textContent || '').trim(); return { tag: (el.tagName || '').toLowerCase(), text, image_urls: Array.from(new Set(imageUrls)), attrs, }; })""" ) return [ NodeSnapshot( tag=item.get("tag", ""), text=item.get("text", ""), image_urls=list(item.get("image_urls", [])), attrs=dict(item.get("attrs", {})), ) for item in payload ] finally: await session.page.close() await session.context.close() self._pages.pop(page_id, None) _runtime = _BrowserRuntime() def open_page(url: str, wait_ms: int = 3000) -> str: """打开页面并返回 page_id。""" return _runtime._run(_runtime._open_page(url, wait_ms)) def query(page_id: str, selector: str) -> list[NodeSnapshot]: """按选择器抓取节点,提取文本、图片 URL 与属性。""" items = _runtime._run(_runtime._query(page_id, selector)) seen_urls: set[str] = set() for item in items: unique_urls: list[str] = [] for url in item.image_urls: if url in seen_urls: continue seen_urls.add(url) unique_urls.append(url) item.image_urls = unique_urls return items