275 lines
8.9 KiB
Python
275 lines
8.9 KiB
Python
"""Playwright 页面抓取工具。
|
|
|
|
对外仅暴露两个动作：
|
|
- open_page(url, wait_ms): 打开页面，返回 page_id
|
|
- query(page_id, selector): 选中节点并提取文字、图片 URL 与属性
|
|
|
|
实现上内部常驻一个 Browser；每次 open_page 创建独立 BrowserContext 和 Page。
|
|
Playwright 只在一个后台线程里运行，但外层 API 保持同步。
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import atexit
|
|
import queue
|
|
import threading
|
|
import uuid
|
|
from concurrent.futures import Future
|
|
from dataclasses import dataclass
|
|
from typing import Any, Callable
|
|
|
|
from playwright.sync_api import Browser, BrowserContext, Page, sync_playwright
|
|
|
|
|
|
@dataclass
|
|
class NodeSnapshot:
|
|
"""页面节点快照。"""
|
|
|
|
tag: str
|
|
text: str
|
|
image_urls: list[str]
|
|
attrs: dict[str, str]
|
|
|
|
|
|
@dataclass
|
|
class _PageSession:
|
|
context: BrowserContext
|
|
page: Page
|
|
|
|
|
|
class _BrowserRuntime:
|
|
"""在单独线程中管理 Playwright 生命周期。"""
|
|
|
|
def __init__(self) -> None:
|
|
self._thread: threading.Thread | None = None
|
|
self._ready = threading.Event()
|
|
self._closed = threading.Event()
|
|
self._job_queue: queue.Queue[
|
|
tuple[Callable[..., Any], tuple[Any, ...], dict[str, Any], Future[Any]]
|
|
] = queue.Queue()
|
|
self._lock = threading.RLock()
|
|
self._playwright = None
|
|
self._browser: Browser | None = None
|
|
self._pages: dict[str, _PageSession] = {}
|
|
atexit.register(self.close)
|
|
|
|
def _thread_main(self) -> None:
|
|
with sync_playwright() as playwright:
|
|
self._playwright = playwright
|
|
self._browser = playwright.chromium.launch(headless=True)
|
|
self._ready.set()
|
|
|
|
while True:
|
|
job = self._job_queue.get()
|
|
if job is None:
|
|
break
|
|
|
|
fn, args, kwargs, fut = job
|
|
try:
|
|
result = fn(*args, **kwargs)
|
|
except BaseException as exc: # pragma: no cover - propagate failures
|
|
fut.set_exception(exc)
|
|
else:
|
|
fut.set_result(result)
|
|
|
|
for session in self._pages.values():
|
|
try:
|
|
session.page.close()
|
|
except Exception:
|
|
pass
|
|
try:
|
|
session.context.close()
|
|
except Exception:
|
|
pass
|
|
self._pages.clear()
|
|
|
|
if self._browser is not None:
|
|
try:
|
|
self._browser.close()
|
|
finally:
|
|
self._browser = None
|
|
|
|
self._playwright = None
|
|
self._closed.set()
|
|
|
|
def _ensure_thread(self) -> None:
|
|
with self._lock:
|
|
if self._thread and self._thread.is_alive():
|
|
return
|
|
|
|
self._thread = threading.Thread(
|
|
target=self._thread_main,
|
|
name="offerpie-playwright",
|
|
daemon=True,
|
|
)
|
|
self._thread.start()
|
|
self._ready.wait()
|
|
|
|
def _run(self, fn: Callable[..., Any], *args: Any, **kwargs: Any) -> Any:
|
|
self._ensure_thread()
|
|
fut: Future[Any] = Future()
|
|
self._job_queue.put((fn, args, kwargs, fut))
|
|
return fut.result()
|
|
|
|
def close(self) -> None:
|
|
with self._lock:
|
|
if self._closed.is_set():
|
|
return
|
|
if self._thread is None:
|
|
return
|
|
self._job_queue.put(None)
|
|
self._closed.wait(timeout=10)
|
|
|
|
def _ensure_browser(self) -> Browser:
|
|
if self._browser is None:
|
|
raise RuntimeError("Browser runtime is not ready.")
|
|
return self._browser
|
|
|
|
def _open_page_impl(self, url: str, wait_ms: int) -> str:
|
|
browser = self._ensure_browser()
|
|
context = browser.new_context()
|
|
page = context.new_page()
|
|
page.goto(url, wait_until="domcontentloaded", timeout=30000)
|
|
if wait_ms > 0:
|
|
page.wait_for_timeout(wait_ms)
|
|
|
|
page_id = uuid.uuid4().hex
|
|
self._pages[page_id] = _PageSession(context=context, page=page)
|
|
return page_id
|
|
|
|
def _query_impl(self, page_id: str, selector: str) -> list[NodeSnapshot]:
|
|
session = self._pages.get(page_id)
|
|
if session is None:
|
|
raise KeyError(f"Unknown page_id: {page_id}")
|
|
|
|
locator = session.page.locator(selector)
|
|
payload = locator.evaluate_all(
|
|
r"""(elements) => elements.map((el) => {
|
|
const attrs = {};
|
|
for (const attr of Array.from(el.attributes || [])) {
|
|
attrs[attr.name] = attr.value;
|
|
}
|
|
|
|
const imageUrls = [];
|
|
const addImageUrl = (url) => {
|
|
if (!url) return;
|
|
try {
|
|
const absUrl = new URL(url, location.href).href;
|
|
if (
|
|
absUrl.startsWith('data:') ||
|
|
absUrl.startsWith('blob:') ||
|
|
absUrl === 'about:blank'
|
|
) {
|
|
return;
|
|
}
|
|
imageUrls.push(absUrl);
|
|
} catch {
|
|
if (
|
|
url.startsWith('data:') ||
|
|
url.startsWith('blob:') ||
|
|
url === 'about:blank'
|
|
) {
|
|
return;
|
|
}
|
|
imageUrls.push(url);
|
|
}
|
|
};
|
|
|
|
const resolveImageUrl = (img) => {
|
|
const candidates = [
|
|
img.getAttribute('data-src'),
|
|
img.getAttribute('data-original'),
|
|
img.getAttribute('data-croporisrc'),
|
|
img.currentSrc,
|
|
img.src,
|
|
img.getAttribute('src'),
|
|
];
|
|
|
|
const srcset = img.getAttribute('srcset');
|
|
if (srcset) {
|
|
for (const part of srcset.split(',')) {
|
|
const candidate = part.trim().split(/\s+/)[0];
|
|
candidates.push(candidate);
|
|
}
|
|
}
|
|
|
|
for (const candidate of candidates) {
|
|
if (!candidate) continue;
|
|
if (
|
|
candidate.startsWith('data:') ||
|
|
candidate.startsWith('blob:') ||
|
|
candidate === 'about:blank'
|
|
) {
|
|
continue;
|
|
}
|
|
try {
|
|
const absUrl = new URL(candidate, location.href).href;
|
|
if (
|
|
absUrl.startsWith('data:') ||
|
|
absUrl.startsWith('blob:') ||
|
|
absUrl === 'about:blank'
|
|
) {
|
|
continue;
|
|
}
|
|
return absUrl;
|
|
} catch {
|
|
return candidate;
|
|
}
|
|
}
|
|
|
|
return null;
|
|
};
|
|
|
|
if (el.tagName === 'IMG') {
|
|
addImageUrl(resolveImageUrl(el));
|
|
}
|
|
|
|
for (const img of Array.from(el.querySelectorAll('img'))) {
|
|
addImageUrl(resolveImageUrl(img));
|
|
}
|
|
|
|
const text = (el.innerText || el.textContent || '').trim();
|
|
|
|
return {
|
|
tag: (el.tagName || '').toLowerCase(),
|
|
text,
|
|
image_urls: Array.from(new Set(imageUrls)),
|
|
attrs,
|
|
};
|
|
})"""
|
|
)
|
|
return [
|
|
NodeSnapshot(
|
|
tag=item.get("tag", ""),
|
|
text=item.get("text", ""),
|
|
image_urls=list(item.get("image_urls", [])),
|
|
attrs=dict(item.get("attrs", {})),
|
|
)
|
|
for item in payload
|
|
]
|
|
|
|
|
|
_runtime = _BrowserRuntime()
|
|
|
|
|
|
def open_page(url: str, wait_ms: int = 3000) -> str:
|
|
"""打开页面并返回 page_id。"""
|
|
return _runtime._run(_runtime._open_page_impl, url, wait_ms)
|
|
|
|
|
|
def query(page_id: str, selector: str) -> list[NodeSnapshot]:
|
|
"""按选择器抓取节点，提取文本、图片 URL 与属性。"""
|
|
items = _runtime._run(_runtime._query_impl, page_id, selector)
|
|
seen_urls: set[str] = set()
|
|
|
|
for item in items:
|
|
unique_urls: list[str] = []
|
|
for url in item.image_urls:
|
|
if url in seen_urls:
|
|
continue
|
|
seen_urls.add(url)
|
|
unique_urls.append(url)
|
|
item.image_urls = unique_urls
|
|
|
|
return items
|