98 lines
2.8 KiB
Python
98 lines
2.8 KiB
Python
|
# -*- coding: utf-8 -*-
|
|||
|
|
"""
|
|||
|
|
文档解析器 - 支持多种文件格式
|
|||
|
|
"""
|
|||
|
|
import os
|
|||
|
|
import chardet
|
|||
|
|
from config import SUPPORTED_EXTENSIONS
|
|||
|
|
|
|||
|
|
def detect_encoding(file_path):
|
|||
|
|
"""检测文件编码"""
|
|||
|
|
with open(file_path, 'rb') as f:
|
|||
|
|
raw_data = f.read(10000)
|
|||
|
|
result = chardet.detect(raw_data)
|
|||
|
|
return result['encoding'] or 'utf-8'
|
|||
|
|
|
|||
|
|
def parse_txt(file_path):
|
|||
|
|
"""解析纯文本文件"""
|
|||
|
|
encoding = detect_encoding(file_path)
|
|||
|
|
try:
|
|||
|
|
with open(file_path, 'r', encoding=encoding, errors='ignore') as f:
|
|||
|
|
return f.read()
|
|||
|
|
except Exception as e:
|
|||
|
|
print(f"解析TXT文件失败 {file_path}: {e}")
|
|||
|
|
return ""
|
|||
|
|
|
|||
|
|
def parse_md(file_path):
|
|||
|
|
"""解析Markdown文件"""
|
|||
|
|
return parse_txt(file_path)
|
|||
|
|
|
|||
|
|
def parse_pdf(file_path):
|
|||
|
|
"""解析PDF文件（支持大文件）"""
|
|||
|
|
try:
|
|||
|
|
from PyPDF2 import PdfReader
|
|||
|
|
|
|||
|
|
file_size = os.path.getsize(file_path)
|
|||
|
|
file_size_mb = file_size / (1024 * 1024)
|
|||
|
|
print(f" PDF文件大小: {file_size_mb:.1f} MB")
|
|||
|
|
|
|||
|
|
reader = PdfReader(file_path)
|
|||
|
|
total_pages = len(reader.pages)
|
|||
|
|
print(f" PDF总页数: {total_pages}")
|
|||
|
|
|
|||
|
|
text_parts = []
|
|||
|
|
for i, page in enumerate(reader.pages):
|
|||
|
|
if (i + 1) % 50 == 0 or i == 0:
|
|||
|
|
print(f" 解析进度: {i + 1}/{total_pages} 页")
|
|||
|
|
try:
|
|||
|
|
text = page.extract_text()
|
|||
|
|
if text:
|
|||
|
|
text_parts.append(text)
|
|||
|
|
except Exception as e:
|
|||
|
|
print(f" 警告: 第 {i + 1} 页解析失败: {e}")
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
print(f" PDF解析完成，提取文本 {len(''.join(text_parts))} 字符")
|
|||
|
|
return "\n".join(text_parts)
|
|||
|
|
except Exception as e:
|
|||
|
|
print(f"解析PDF文件失败 {file_path}: {e}")
|
|||
|
|
return ""
|
|||
|
|
|
|||
|
|
def parse_docx(file_path):
|
|||
|
|
"""解析Word文档"""
|
|||
|
|
try:
|
|||
|
|
from docx import Document
|
|||
|
|
doc = Document(file_path)
|
|||
|
|
text_parts = []
|
|||
|
|
for para in doc.paragraphs:
|
|||
|
|
if para.text.strip():
|
|||
|
|
text_parts.append(para.text)
|
|||
|
|
return "\n".join(text_parts)
|
|||
|
|
except Exception as e:
|
|||
|
|
print(f"解析DOCX文件失败 {file_path}: {e}")
|
|||
|
|
return ""
|
|||
|
|
|
|||
|
|
def parse_document(file_path):
|
|||
|
|
"""根据文件类型解析文档"""
|
|||
|
|
ext = os.path.splitext(file_path)[1].lower()
|
|||
|
|
|
|||
|
|
if ext not in SUPPORTED_EXTENSIONS:
|
|||
|
|
print(f"不支持的文件类型: {ext}")
|
|||
|
|
return ""
|
|||
|
|
|
|||
|
|
parsers = {
|
|||
|
|
'.txt': parse_txt,
|
|||
|
|
'.md': parse_md,
|
|||
|
|
'.pdf': parse_pdf,
|
|||
|
|
'.docx': parse_docx,
|
|||
|
|
'.doc': parse_docx,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
parser = parsers.get(ext, parse_txt)
|
|||
|
|
return parser(file_path)
|
|||
|
|
|
|||
|
|
def is_supported_file(filename):
|
|||
|
|
"""检查文件是否支持"""
|
|||
|
|
ext = os.path.splitext(filename)[1].lower()
|
|||
|
|
return ext in SUPPORTED_EXTENSIONS
|