From 39e162ae0fbd5d44ad648994aef2cc62c72b01a5 Mon Sep 17 00:00:00 2001 From: zk Date: Fri, 24 Jul 2026 16:56:57 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=B5=8F=E8=A7=88=E5=99=A8?= =?UTF-8?q?=E6=96=B9=E6=A1=88=E4=B8=BA=E5=90=8C=E6=AD=A5=E6=96=B9=E6=A1=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tool/browser.py | 296 ++++++++++++++++++++++++++---------------------- 1 file changed, 158 insertions(+), 138 deletions(-) diff --git a/tool/browser.py b/tool/browser.py index f80d51a..3dae968 100644 --- a/tool/browser.py +++ b/tool/browser.py @@ -4,18 +4,21 @@ - open_page(url, wait_ms): 打开页面，返回 page_id - query(page_id, selector): 选中节点并提取文字、图片 URL 与属性 -实现上内部常驻一个 Browser；每次 open_page 创建独立 BrowserContext 和 Page， -query 完成后自动释放对应资源。 +实现上内部常驻一个 Browser；每次 open_page 创建独立 BrowserContext 和 Page。 +Playwright 只在一个后台线程里运行，但外层 API 保持同步。 """ from __future__ import annotations -import asyncio +import atexit +import queue import threading import uuid +from concurrent.futures import Future from dataclasses import dataclass +from typing import Any, Callable -from playwright.async_api import Browser, BrowserContext, Page, async_playwright +from playwright.sync_api import Browser, BrowserContext, Page, sync_playwright @dataclass @@ -35,41 +38,59 @@ class _PageSession: class _BrowserRuntime: - """在后台事件循环中管理 Playwright 生命周期。""" + """在单独线程中管理 Playwright 生命周期。""" def __init__(self) -> None: self._thread: threading.Thread | None = None - self._loop: asyncio.AbstractEventLoop | None = None self._ready = threading.Event() + self._closed = threading.Event() + self._job_queue: queue.Queue[ + tuple[Callable[..., Any], tuple[Any, ...], dict[str, Any], Future[Any]] + ] = queue.Queue() self._lock = threading.RLock() self._playwright = None self._browser: Browser | None = None self._pages: dict[str, _PageSession] = {} + atexit.register(self.close) def _thread_main(self) -> None: - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - self._loop = loop - self._ready.set() - loop.run_forever() + with sync_playwright() as playwright: + self._playwright = playwright + self._browser = playwright.chromium.launch(headless=True) + self._ready.set() - pending = asyncio.all_tasks(loop) - for task in pending: - task.cancel() + while True: + job = self._job_queue.get() + if job is None: + break - if pending: - loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True)) + fn, args, kwargs, fut = job + try: + result = fn(*args, **kwargs) + except BaseException as exc: # pragma: no cover - propagate failures + fut.set_exception(exc) + else: + fut.set_result(result) - if self._browser is not None: - loop.run_until_complete(self._browser.close()) - self._browser = None + for session in self._pages.values(): + try: + session.page.close() + except Exception: + pass + try: + session.context.close() + except Exception: + pass + self._pages.clear() + + if self._browser is not None: + try: + self._browser.close() + finally: + self._browser = None - if self._playwright is not None: - loop.run_until_complete(self._playwright.stop()) self._playwright = None - - loop.run_until_complete(loop.shutdown_asyncgens()) - loop.close() + self._closed.set() def _ensure_thread(self) -> None: with self._lock: @@ -84,149 +105,148 @@ class _BrowserRuntime: self._thread.start() self._ready.wait() - def _run(self, coro): + def _run(self, fn: Callable[..., Any], *args: Any, **kwargs: Any) -> Any: self._ensure_thread() - assert self._loop is not None - future = asyncio.run_coroutine_threadsafe(coro, self._loop) - return future.result() + fut: Future[Any] = Future() + self._job_queue.put((fn, args, kwargs, fut)) + return fut.result() - async def _ensure_browser(self) -> Browser: - if self._browser is not None: - return self._browser + def close(self) -> None: + with self._lock: + if self._closed.is_set(): + return + if self._thread is None: + return + self._job_queue.put(None) + self._closed.wait(timeout=10) - if self._playwright is None: - self._playwright = await async_playwright().start() - - self._browser = await self._playwright.chromium.launch(headless=True) + def _ensure_browser(self) -> Browser: + if self._browser is None: + raise RuntimeError("Browser runtime is not ready.") return self._browser - async def _open_page(self, url: str, wait_ms: int) -> str: - browser = await self._ensure_browser() - context = await browser.new_context() - page = await context.new_page() - await page.goto(url, wait_until="domcontentloaded", timeout=30000) + def _open_page_impl(self, url: str, wait_ms: int) -> str: + browser = self._ensure_browser() + context = browser.new_context() + page = context.new_page() + page.goto(url, wait_until="domcontentloaded", timeout=30000) if wait_ms > 0: - await page.wait_for_timeout(wait_ms) + page.wait_for_timeout(wait_ms) page_id = uuid.uuid4().hex self._pages[page_id] = _PageSession(context=context, page=page) return page_id - async def _query(self, page_id: str, selector: str) -> list[NodeSnapshot]: + def _query_impl(self, page_id: str, selector: str) -> list[NodeSnapshot]: session = self._pages.get(page_id) if session is None: raise KeyError(f"Unknown page_id: {page_id}") - try: - locator = session.page.locator(selector) - payload = await locator.evaluate_all( - r"""(elements) => elements.map((el) => { - const attrs = {}; - for (const attr of Array.from(el.attributes || [])) { - attrs[attr.name] = attr.value; + locator = session.page.locator(selector) + payload = locator.evaluate_all( + r"""(elements) => elements.map((el) => { + const attrs = {}; + for (const attr of Array.from(el.attributes || [])) { + attrs[attr.name] = attr.value; + } + + const imageUrls = []; + const addImageUrl = (url) => { + if (!url) return; + try { + const absUrl = new URL(url, location.href).href; + if ( + absUrl.startsWith('data:') || + absUrl.startsWith('blob:') || + absUrl === 'about:blank' + ) { + return; + } + imageUrls.push(absUrl); + } catch { + if ( + url.startsWith('data:') || + url.startsWith('blob:') || + url === 'about:blank' + ) { + return; + } + imageUrls.push(url); + } + }; + + const resolveImageUrl = (img) => { + const candidates = [ + img.getAttribute('data-src'), + img.getAttribute('data-original'), + img.getAttribute('data-croporisrc'), + img.currentSrc, + img.src, + img.getAttribute('src'), + ]; + + const srcset = img.getAttribute('srcset'); + if (srcset) { + for (const part of srcset.split(',')) { + const candidate = part.trim().split(/\s+/)[0]; + candidates.push(candidate); + } } - const imageUrls = []; - const addImageUrl = (url) => { - if (!url) return; + for (const candidate of candidates) { + if (!candidate) continue; + if ( + candidate.startsWith('data:') || + candidate.startsWith('blob:') || + candidate === 'about:blank' + ) { + continue; + } try { - const absUrl = new URL(url, location.href).href; + const absUrl = new URL(candidate, location.href).href; if ( absUrl.startsWith('data:') || absUrl.startsWith('blob:') || absUrl === 'about:blank' - ) { - return; - } - imageUrls.push(absUrl); - } catch { - if ( - url.startsWith('data:') || - url.startsWith('blob:') || - url === 'about:blank' - ) { - return; - } - imageUrls.push(url); - } - }; - - const resolveImageUrl = (img) => { - const candidates = [ - img.getAttribute('data-src'), - img.getAttribute('data-original'), - img.getAttribute('data-croporisrc'), - img.currentSrc, - img.src, - img.getAttribute('src'), - ]; - - const srcset = img.getAttribute('srcset'); - if (srcset) { - for (const part of srcset.split(',')) { - const candidate = part.trim().split(/\s+/)[0]; - candidates.push(candidate); - } - } - - for (const candidate of candidates) { - if (!candidate) continue; - if ( - candidate.startsWith('data:') || - candidate.startsWith('blob:') || - candidate === 'about:blank' ) { continue; } - try { - const absUrl = new URL(candidate, location.href).href; - if ( - absUrl.startsWith('data:') || - absUrl.startsWith('blob:') || - absUrl === 'about:blank' - ) { - continue; - } - return absUrl; - } catch { - return candidate; - } + return absUrl; + } catch { + return candidate; } - - return null; - }; - - if (el.tagName === 'IMG') { - addImageUrl(resolveImageUrl(el)); } - for (const img of Array.from(el.querySelectorAll('img'))) { - addImageUrl(resolveImageUrl(img)); - } + return null; + }; - const text = (el.innerText || el.textContent || '').trim(); + if (el.tagName === 'IMG') { + addImageUrl(resolveImageUrl(el)); + } - return { - tag: (el.tagName || '').toLowerCase(), - text, - image_urls: Array.from(new Set(imageUrls)), - attrs, - }; - })""" + for (const img of Array.from(el.querySelectorAll('img'))) { + addImageUrl(resolveImageUrl(img)); + } + + const text = (el.innerText || el.textContent || '').trim(); + + return { + tag: (el.tagName || '').toLowerCase(), + text, + image_urls: Array.from(new Set(imageUrls)), + attrs, + }; + })""" + ) + return [ + NodeSnapshot( + tag=item.get("tag", ""), + text=item.get("text", ""), + image_urls=list(item.get("image_urls", [])), + attrs=dict(item.get("attrs", {})), ) - return [ - NodeSnapshot( - tag=item.get("tag", ""), - text=item.get("text", ""), - image_urls=list(item.get("image_urls", [])), - attrs=dict(item.get("attrs", {})), - ) - for item in payload - ] - finally: - await session.page.close() - await session.context.close() - self._pages.pop(page_id, None) + for item in payload + ] _runtime = _BrowserRuntime() @@ -234,12 +254,12 @@ _runtime = _BrowserRuntime() def open_page(url: str, wait_ms: int = 3000) -> str: """打开页面并返回 page_id。""" - return _runtime._run(_runtime._open_page(url, wait_ms)) + return _runtime._run(_runtime._open_page_impl, url, wait_ms) def query(page_id: str, selector: str) -> list[NodeSnapshot]: """按选择器抓取节点，提取文本、图片 URL 与属性。""" - items = _runtime._run(_runtime._query(page_id, selector)) + items = _runtime._run(_runtime._query_impl, page_id, selector) seen_urls: set[str] = set() for item in items: