重构简历导入
This commit is contained in:
zk
@@ -1,92 +0,0 @@
|
||||
"""文件解析工具
|
||||
|
||||
将上传的简历文件解析为「文本单元数组」，供 AI 按行号定位 description/summary，避免照抄长文本。
|
||||
- PDF：使用 LiteParse（JSON 模式）按文本块的阅读顺序提取，每个文本块作为一个单元。
|
||||
关闭 OCR —— 文本型简历无需 OCR，且 OCR 默认会联网下载字库导致严重阻塞。
|
||||
- Word(.docx)：按段落（paragraph）切分，表格行用 \t 拼成一个单元。
|
||||
- TXT / Markdown(.md)：自动检测编码，按换行切分。
|
||||
"""
|
||||
|
||||
import io
|
||||
from collections import Counter
|
||||
|
||||
from docx import Document
|
||||
from liteparse import LiteParse
|
||||
|
||||
from app.core.logger import log
|
||||
|
||||
# LiteParse 解析器复用实例（关闭 OCR、静默日志）。线程安全由调用方的 to_thread 串行保证。
|
||||
_PDF_PARSER = LiteParse(output_format="json", ocr_enabled=False, quiet=True)
|
||||
|
||||
|
||||
def _drop_repeated(segments: list[str], threshold: int = 3) -> list[str]:
|
||||
"""过滤重复出现的行（水印/页眉页脚特征）：完全相同且出现次数 >= threshold 的行整体丢弃"""
|
||||
counts = Counter(segments)
|
||||
return [s for s in segments if counts[s] < threshold]
|
||||
|
||||
|
||||
def _pdf_segments(content: bytes) -> list[str]:
|
||||
"""解析 PDF：使用 LiteParse 按阅读顺序遍历每页文本块，每块作为一个独立单元，过滤空行。
|
||||
|
||||
不在预处理阶段合并行——是否合并/分段/排除噪声全部交给 AI 的行号区间决定。
|
||||
"""
|
||||
result = _PDF_PARSER.parse(content)
|
||||
segments: list[str] = []
|
||||
for page in result.pages:
|
||||
for item in page.text_items:
|
||||
text = item.text.strip()
|
||||
if text:
|
||||
segments.append(text)
|
||||
return segments
|
||||
|
||||
|
||||
def _docx_segments(content: bytes) -> list[str]:
|
||||
"""解析 Word (.docx)，按段落切分为单元，表格行用 \t 拼为一个单元"""
|
||||
try:
|
||||
doc = Document(io.BytesIO(content))
|
||||
except Exception:
|
||||
raise ValueError("无法解析该 Word 文件，请确认为有效的 .docx 格式")
|
||||
segments: list[str] = []
|
||||
for para in doc.paragraphs:
|
||||
text = para.text.strip()
|
||||
if text:
|
||||
segments.append(text)
|
||||
for table in doc.tables:
|
||||
for row in table.rows:
|
||||
row_text = "\t".join(cell.text.strip() for cell in row.cells)
|
||||
if row_text.strip():
|
||||
segments.append(row_text)
|
||||
return segments
|
||||
|
||||
|
||||
def _txt_segments(content: bytes) -> list[str]:
|
||||
"""解析纯文本 (.txt / .md)，自动检测编码，按换行切分为单元"""
|
||||
text = None
|
||||
for encoding in ("utf-8", "gbk", "gb2312", "latin-1"):
|
||||
try:
|
||||
text = content.decode(encoding)
|
||||
break
|
||||
except (UnicodeDecodeError, LookupError):
|
||||
continue
|
||||
if text is None:
|
||||
text = content.decode("utf-8", errors="replace")
|
||||
return [line.strip() for line in text.splitlines() if line.strip()]
|
||||
|
||||
|
||||
# 后缀 → 解析函数。新增格式只需在此登记一行。
|
||||
_PARSERS = {
|
||||
".pdf": _pdf_segments,
|
||||
".docx": _docx_segments,
|
||||
".txt": _txt_segments,
|
||||
".md": _txt_segments,
|
||||
}
|
||||
|
||||
|
||||
def parse_to_segments(filename: str, content: bytes) -> list[str]:
|
||||
"""根据文件名后缀选择解析器，返回去重后的「文本单元数组」"""
|
||||
suffix = filename[filename.rfind("."):].lower() if "." in filename else ""
|
||||
log.info(f"解析文件: {filename}，类型: {suffix}")
|
||||
parser = _PARSERS.get(suffix)
|
||||
if parser is None:
|
||||
raise ValueError(f"不支持的文件类型: {suffix}，支持: {', '.join(_PARSERS)}")
|
||||
return _drop_repeated(parser(content))
|
||||
@@ -0,0 +1,95 @@
|
||||
"""简历文字处理
|
||||
|
||||
将上传的简历文件提取为纯文本，供后续 AI 重写/结构化使用。
|
||||
支持格式：.txt / .md / .pdf / .docx
|
||||
所有解析均为异步（阻塞解析下沉到线程池，避免阻塞事件循环）。
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
|
||||
import fitz # PyMuPDF
|
||||
from docx import Document
|
||||
|
||||
from app.core.asserts import Assert
|
||||
from app.core.logger import log
|
||||
|
||||
# 支持的简历文件类型
|
||||
SUPPORTED_EXTENSIONS = (".txt", ".md", ".pdf", ".docx")
|
||||
|
||||
# 纯文本解码尝试的编码顺序
|
||||
_TEXT_ENCODINGS = ("utf-8", "gbk", "gb2312", "latin-1")
|
||||
|
||||
|
||||
async def _parse_txt(content: bytes) -> str:
|
||||
"""解析纯文本 (.txt / .md)：自动探测编码，返回全文文本"""
|
||||
|
||||
def _decode() -> str:
|
||||
for encoding in _TEXT_ENCODINGS:
|
||||
try:
|
||||
return content.decode(encoding)
|
||||
except (UnicodeDecodeError, LookupError):
|
||||
continue
|
||||
# 全部失败时以替换字符兜底，保证不抛异常
|
||||
return content.decode("utf-8", errors="replace")
|
||||
|
||||
return await asyncio.to_thread(_decode)
|
||||
|
||||
|
||||
async def _parse_pdf(content: bytes) -> str:
|
||||
"""解析 PDF：使用 PyMuPDF 逐页 get_text() 拼接为全文文本"""
|
||||
|
||||
def _extract() -> str:
|
||||
parts: list[str] = []
|
||||
with fitz.open(stream=content, filetype="pdf") as doc:
|
||||
for page in doc:
|
||||
parts.append(page.get_text())
|
||||
return "\n".join(parts)
|
||||
|
||||
return await asyncio.to_thread(_extract)
|
||||
|
||||
|
||||
async def _parse_docx(content: bytes) -> str:
|
||||
"""解析 Word (.docx)：拼接段落文本，表格行用 \t 连接，返回全文文本"""
|
||||
|
||||
def _extract() -> str:
|
||||
try:
|
||||
doc = Document(io.BytesIO(content))
|
||||
except Exception:
|
||||
raise ValueError("无法解析该 Word 文件，请确认为有效的 .docx 格式")
|
||||
parts: list[str] = []
|
||||
for para in doc.paragraphs:
|
||||
text = para.text.strip()
|
||||
if text:
|
||||
parts.append(text)
|
||||
for table in doc.tables:
|
||||
for row in table.rows:
|
||||
row_text = "\t".join(cell.text.strip() for cell in row.cells)
|
||||
if row_text.strip():
|
||||
parts.append(row_text)
|
||||
return "\n".join(parts)
|
||||
|
||||
return await asyncio.to_thread(_extract)
|
||||
|
||||
|
||||
# 后缀 → 解析函数。新增格式只需在此登记一行，并同步 SUPPORTED_EXTENSIONS。
|
||||
_PARSERS = {
|
||||
".txt": _parse_txt,
|
||||
".md": _parse_txt,
|
||||
".pdf": _parse_pdf,
|
||||
".docx": _parse_docx,
|
||||
}
|
||||
|
||||
|
||||
async def extract_text(filename: str, content: bytes) -> str:
|
||||
"""简历文字提取统一入口：先校验文件类型受支持，再按后缀路由提取全文文本"""
|
||||
suffix = filename[filename.rfind("."):].lower() if "." in filename else ""
|
||||
Assert.is_true(
|
||||
suffix in SUPPORTED_EXTENSIONS,
|
||||
f"不支持的文件类型: {suffix or '未知'}，支持: {', '.join(SUPPORTED_EXTENSIONS)}",
|
||||
)
|
||||
|
||||
log.info(f"开始提取简历文本: {filename}，类型: {suffix}")
|
||||
text = await _PARSERS[suffix](content)
|
||||
log.info(f"简历文本提取完成: {filename}，字符数: {len(text)}")
|
||||
return text
|
||||
Reference in New Issue
Block a user