背景介绍
在日常工作和学习中，我们经常需要在大量文档中查找特定内容。Windows自带的搜索功能效率低下，而第三方工具往往功能冗余。本文将介绍如何使用Python开发一个轻量级本地文件内容检索工具，支持多种文档格式的全文搜索，并实现关键词高亮显示。
系统设计
核心架构
graph TD
A[文件扫描] --> B[内容提取]
B --> C[索引构建]
C --> D[查询处理]
D --> E[结果排序]
E --> F[高亮输出]
技术选型
- 文件解析：textract + pypdf2
- 索引结构：倒排索引
- 搜索算法：TF-IDF权重
- 结果展示：命令行彩色输出
完整代码实现
import os
import re
from collections import defaultdict
from pathlib import Path
import textract
from math import log
from termcolor import colored
class FileSearcher:
def __init__(self):
self.index = defaultdict(dict) # 倒排索引 {term: {doc_id: tf}}
self.doc_info = {} # 文档元数据 {doc_id: {path, length}}
self.total_docs = 0
def build_index(self, directory):
"""构建全文索引"""
supported_ext = ['.txt', '.pdf', '.docx']
for i, filepath in enumerate(Path(directory).rglob('*')):
if filepath.suffix.lower() in supported_ext:
try:
content = self._extract_content(filepath)
self._add_to_index(filepath, content, i)
self.total_docs += 1
except Exception as e:
print(f"解析失败: {filepath} - {str(e)}")
def _extract_content(self, filepath):
"""提取文件内容"""
text = textract.process(str(filepath)).decode('utf-8')
return re.sub(r'\s+', ' ', text).strip()
def _add_to_index(self, filepath, content, doc_id):
"""添加文档到索引"""
words = re.findall(r'\w+', content.lower())
self.doc_info[doc_id] = {
'path': str(filepath),
'length': len(words)
}
# 计算词频
term_freq = defaultdict(int)
for word in words:
term_freq[word] += 1
# 更新倒排索引
for term, tf in term_freq.items():
self.index[term][doc_id] = tf
def search(self, query, top_n=5):
"""执行搜索"""
terms = re.findall(r'\w+', query.lower())
if not terms:
return []
# 计算文档相关性得分
doc_scores = defaultdict(float)
for term in terms:
if term in self.index:
idf = log(self.total_docs / len(self.index[term]))
for doc_id, tf in self.index[term].items():
tf_norm = tf / self.doc_info[doc_id]['length']
doc_scores[doc_id] += tf_norm * idf
# 排序结果
sorted_results = sorted(doc_scores.items(),
key=lambda x: x[1],
reverse=True)[:top_n]
# 准备输出
results = []
for doc_id, score in sorted_results:
path = self.doc_info[doc_id]['path']
preview = self._get_preview(path, terms)
results.append({
'path': path,
'score': round(score * 100),
'preview': preview
})
return results
def _get_preview(self, filepath, terms):
"""获取匹配内容片段"""
content = self._extract_content(filepath)
sentences = re.split(r'[.!?]', content)
# 查找包含最多关键词的句子
best_sentence = ""
max_count = 0
for sentence in sentences:
count = sum(1 for term in terms
if term in sentence.lower())
if count > max_count:
max_count = count
best_sentence = sentence.strip()
# 高亮关键词
if best_sentence:
for term in terms:
best_sentence = re.sub(
f'({term})',
colored(r'\1', 'red', attrs=['bold']),
best_sentence,
flags=re.IGNORECASE
)
return best_sentence[:200] + "..."
return ""
def main():
print("=== 本地文件内容检索工具 ===")
searcher = FileSearcher()
# 构建索引
directory = input("请输入要索引的目录路径: ")
print("正在构建索引...")
searcher.build_index(directory)
print(f"索引完成，共处理 {searcher.total_docs} 个文档")
# 搜索循环
while True:
query = input("\n请输入搜索关键词(输入q退出): ")
if query.lower() == 'q':
break
results = searcher.search(query)
if not results:
print("未找到匹配结果")
continue
print("\n[搜索结果]")
for i, result in enumerate(results, 1):
print(f"{i}. {result['path']} (匹配度: {result['score']}%)")
print(f" {result['preview']}\n")
if __name__ == "__main__":
main()
关键技术解析
1. 倒排索引实现
self.index = defaultdict(dict) # 结构: {term: {doc_id: tf}}
self.doc_info = {} # 存储文档元数据
def _add_to_index(self, filepath, content, doc_id):
words = re.findall(r'\w+', content.lower())
term_freq = defaultdict(int)
for word in words:
term_freq[word] += 1
for term, tf in term_freq.items():
self.index[term][doc_id] = tf
2. TF-IDF相关性计算
idf = log(self.total_docs / len(self.index[term]))
for doc_id, tf in self.index[term].items():
tf_norm = tf / self.doc_info[doc_id]['length']
doc_scores[doc_id] += tf_norm * idf
3. 关键词高亮
for term in terms:
best_sentence = re.sub(
f'({term})',
colored(r'\1', 'red', attrs=['bold']),
best_sentence,
flags=re.IGNORECASE
)
使用示例
- 安装依赖：
pip install textract pypdf2 termcolor
- 运行程序：
python file_searcher.py
- 示例交互：
=== 本地文件内容检索工具 ===
请输入要索引的目录路径: ~/Documents
正在构建索引...
索引完成，共处理 127 个文档
请输入搜索关键词(输入q退出): Python 文件处理
[搜索结果]
1. /home/user/Documents/demo.py (匹配度: 85%)
实现Python文件处理的示例代码，包含read()和write()方法...
2. /home/user/Documents/notes.txt (匹配度: 72%)
文件处理注意事项：Python的with语句可以自动关闭文件...
扩展功能
1. 添加GUI界面
import tkinter as tk
from tkinter import ttk
class SearchGUI:
def __init__(self, searcher):
self.searcher = searcher
self.root = tk.Tk()
self._setup_ui()
def _setup_ui(self):
self.root.title("文件搜索工具")
ttk.Label(self.root, text="搜索:").pack()
self.entry = ttk.Entry(self.root, width=50)
self.entry.pack()
ttk.Button(self.root, text="搜索", command=self._search).pack()
self.results = tk.Text(self.root)
self.results.pack()
2. 支持正则表达式
def search(self, query, use_regex=False):
if use_regex:
try:
terms = [query] # 将整个正则作为单个term
except re.error:
return []
else:
terms = re.findall(r'\w+', query.lower())
3. 拼写纠正
from difflib import get_close_matches
def correct_spelling(self, term):
vocab = list(self.index.keys())
suggestions = get_close_matches(term, vocab, n=1)
return suggestions[0] if suggestions else term
项目总结
本工具实现了以下核心价值：
1. 高效搜索：倒排索引+TF-IDF算法
2. 格式支持：txt/pdf/docx全覆盖
3. 直观展示：命令行高亮结果
4. 轻量易用：纯Python实现，无外部依赖
优化建议：
1. 添加索引持久化功能
2. 实现增量索引更新
3. 支持更多文档格式
完整项目代码已开源在GitHub：[项目链接]