You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
212 lines
7.5 KiB
212 lines
7.5 KiB
5 months ago
|
from docx import Document
|
||
|
|
from paddlenlp import Taskflow
|
||
|
|
from pprint import pprint
|
||
|
|
from qwen_agent.agents import Assistant
|
||
|
|
import re
|
||
|
|
import json_repair
|
||
|
|
import time
|
||
|
|
import math
|
||
|
|
from docx.opc.pkgreader import _SerializedRelationships, _SerializedRelationship
|
||
|
|
from docx.opc.oxml import parse_xml
|
||
|
|
|
||
|
|
|
||
|
|
def load_from_xml_v2(baseURI, rels_item_xml):
|
||
|
|
"""
|
||
|
|
Return |_SerializedRelationships| instance loaded with the
|
||
|
|
relationships contained in *rels_item_xml*. Returns an empty
|
||
|
|
collection if *rels_item_xml* is |None|.
|
||
|
|
"""
|
||
|
|
srels = _SerializedRelationships()
|
||
|
|
if rels_item_xml is not None:
|
||
|
|
rels_elm = parse_xml(rels_item_xml)
|
||
|
|
for rel_elm in rels_elm.Relationship_lst:
|
||
|
|
if rel_elm.target_ref in ('../NULL', 'NULL'):
|
||
|
|
continue
|
||
|
|
srels._srels.append(_SerializedRelationship(baseURI, rel_elm))
|
||
|
|
return srels
|
||
|
|
|
||
|
|
|
||
|
|
_SerializedRelationships.load_from_xml = load_from_xml_v2
|
||
|
|
|
||
|
|
|
||
|
|
import logging
|
||
|
|
import logging.config
|
||
|
|
|
||
|
|
log_config = {
|
||
|
|
'version': 1,
|
||
|
|
'disable_existing_loggers': False,
|
||
|
|
'formatters': {
|
||
|
|
'standard': {
|
||
|
|
'format': '%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
||
|
|
},
|
||
|
|
},
|
||
|
|
'handlers': {
|
||
|
|
'console': {
|
||
|
|
'class': 'logging.StreamHandler',
|
||
|
|
'formatter': 'standard',
|
||
|
|
'level': logging.INFO,
|
||
|
|
},
|
||
|
|
'file': {
|
||
|
|
'class': 'logging.FileHandler',
|
||
|
|
'filename': 'Logger.log',
|
||
|
|
'formatter': 'standard',
|
||
|
|
'level': logging.INFO,
|
||
|
|
},
|
||
|
|
},
|
||
|
|
'loggers': {
|
||
|
|
'': {
|
||
|
|
'handlers': ['console', 'file'],
|
||
|
|
'level': logging.INFO,
|
||
|
|
'propagate': True,
|
||
|
|
},
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
logging.config.dictConfig(log_config)
|
||
|
|
|
||
|
|
logger = logging.getLogger("checkPlaceName")
|
||
|
|
|
||
|
|
prompt='''
|
||
|
|
.上述文本判断地名是否正确，你可以使用工具利用互联网查询，你只能在[正确,错误,简称,未知]三种选项中选择答案,回答格式[{“placeName”:“地名”,"回答":"答案"},{“placeName”:“地名”,"回答":"答案"}]，不做过多的解释,严格按回答格式作答;
|
||
|
|
不做过多的解释,严格按回答格式作答;
|
||
|
|
'''
|
||
|
|
# prompt='''
|
||
|
|
# .请回答以上问题，
|
||
|
|
# ,回答格式[{“placeName”:"原文","回答":"答案"},{“placeName”:"原文","回答":"答案"}]，不做过多的解释,严格按回答格式作答;
|
||
|
|
# 不做过多的解释,严格按回答格式作答;
|
||
|
|
# '''
|
||
|
|
llm_cfg = {
|
||
|
|
#'model': 'qwen1.5-72b-chat',
|
||
|
|
'model':"qwen2-72b",
|
||
|
|
'model_server': 'http://127.0.0.1:1025/v1', # base_url, also known as api_base
|
||
|
|
# 'api_key': 'sk-ea89cf04431645b185990b8af8c9bb13',
|
||
|
|
}
|
||
|
|
bot = Assistant(llm=llm_cfg,
|
||
|
|
name='Assistant',
|
||
|
|
# description='使用RAG检索并回答，支持文件类型：PDF/Word/PPT/TXT/HTML。'
|
||
|
|
)
|
||
|
|
#获取全文内容
|
||
|
|
def getDocxToTextAll(docxPath):
|
||
|
|
document = Document(docxPath)
|
||
|
|
# 逐段读取docx文档的内容
|
||
|
|
levelList=[]
|
||
|
|
words=[]
|
||
|
|
addStart = False
|
||
|
|
levelText=""
|
||
|
|
i = 0
|
||
|
|
for paragraph in document.paragraphs:
|
||
|
|
# 判断该段落的标题级别
|
||
|
|
# 这里用isTitle()临时代表，具体见下文介绍的方法
|
||
|
|
text = paragraph.text
|
||
|
|
if text.strip():#非空判断
|
||
|
|
# print("非空")
|
||
|
|
words.append(text)
|
||
|
|
# 将所有段落文本拼接成一个字符串，并用换行符分隔
|
||
|
|
text = '\n'.join(words)
|
||
|
|
|
||
|
|
# 将文本写入txt文件
|
||
|
|
with open("checkPlaceName.txt", 'w', encoding='utf-8') as txt_file:
|
||
|
|
txt_file.write(text)
|
||
|
|
|
||
|
|
#得到全文和地名有关的内容
|
||
|
|
def placeNameTask(text):
|
||
|
|
yield "文档地名检查---启动中...."
|
||
|
|
tagTask = Taskflow("ner",device_id=2)
|
||
|
|
batchNum=20
|
||
|
|
sentences = re.split(r'[。\n]', text)
|
||
|
|
# 去掉空字符
|
||
|
|
sentences = [sentence.strip() for sentence in sentences if sentence.strip()]
|
||
|
|
# 计算总字符数
|
||
|
|
total_chars = len(sentences)
|
||
|
|
|
||
|
|
# 计算有多少份
|
||
|
|
num_chunks = math.ceil(total_chars / batchNum)
|
||
|
|
|
||
|
|
# 按batchNum字为一份进行处理
|
||
|
|
chunks = [sentences[i:i + batchNum] for i in range(0, total_chars, batchNum)]
|
||
|
|
placeList = []
|
||
|
|
# 打印每一份的内容
|
||
|
|
for i, chunk in enumerate(chunks):
|
||
|
|
yield f"文档地名检查---文档解析进度:{i + 1}/{num_chunks}"
|
||
|
|
|
||
|
|
wenBen=".".join(chunk)
|
||
|
|
try:
|
||
|
|
res = tagTask(wenBen)
|
||
|
|
except Exception as e:
|
||
|
|
logger.warning(chunk)
|
||
|
|
logger.warning("文档地名检查---解析地名出错",e)
|
||
|
|
continue
|
||
|
|
isplace = False
|
||
|
|
for zuhe in res:
|
||
|
|
# 上一个的地名,这一个还是地名，就和上一个相加代替这个
|
||
|
|
if isplace:
|
||
|
|
name = placeList[len(placeList) - 1]
|
||
|
|
if zuhe[1].find("组织机构类") >= 0 or zuhe[1].find("世界地区类") >= 0: # or zuhe[1] == "ns"
|
||
|
|
isplace = True
|
||
|
|
new_text = zuhe[0].replace("\n", "")
|
||
|
|
placeList[len(placeList) - 1] = name + new_text
|
||
|
|
continue
|
||
|
|
if zuhe[1].find("组织机构类") >= 0 or zuhe[1].find("世界地区类") >= 0:
|
||
|
|
isplace = True
|
||
|
|
new_text = zuhe[0].replace("\n", "")
|
||
|
|
placeList.append(new_text)
|
||
|
|
else:
|
||
|
|
isplace = False
|
||
|
|
# 打印总份数
|
||
|
|
yield "文档地名检查---文档解析完成"
|
||
|
|
placeList=list(dict.fromkeys(placeList))
|
||
|
|
yield placeList
|
||
|
|
#主方法
|
||
|
|
def checkPlaceName(filename):
|
||
|
|
yield f"文档地名检查---开始处理文档..." # 每次生成一个数字就发送
|
||
|
|
try:
|
||
|
|
getDocxToTextAll(filename)
|
||
|
|
except Exception as e:
|
||
|
|
logger.warning(e)
|
||
|
|
yield "文档地名检查---文档无法打开，请检查文档内容"
|
||
|
|
return
|
||
|
|
with open("checkPlaceName.txt", "r",encoding='utf-8') as f:
|
||
|
|
gettext = f.read()
|
||
|
|
yield f"文档地名检查---开始解析文档..." # 每次生成一个数字就发送
|
||
|
|
# propnList=placeNameTask(gettext)
|
||
|
|
for item in placeNameTask(gettext):
|
||
|
|
if isinstance(item, str):
|
||
|
|
yield item
|
||
|
|
else:
|
||
|
|
final_list = item # 获取最终结果
|
||
|
|
propnStr = ",".join(final_list)
|
||
|
|
messages = [{'role': 'user', 'content': [{'text': propnStr + prompt}]}]
|
||
|
|
runList = []
|
||
|
|
yield f"文档地名检查---结果生成中..." # 每次生成一个数字就发送
|
||
|
|
cishu=0
|
||
|
|
for rsp in bot.run(messages):
|
||
|
|
runList.append(rsp)
|
||
|
|
if cishu>3:
|
||
|
|
cishu=0
|
||
|
|
yield "文档地名检查---结果生成中"+'.'*cishu
|
||
|
|
cishu+=1
|
||
|
|
data = runList[len(runList) - 1][0]["content"]
|
||
|
|
parsed_data = json_repair.loads(data.replace('`', ''))
|
||
|
|
error_places=[]
|
||
|
|
# 如果需要进一步操作，例如只关注“正确”的回答
|
||
|
|
for place in parsed_data:
|
||
|
|
try:
|
||
|
|
if place['回答'] == '错误':
|
||
|
|
error_places.append(place)
|
||
|
|
except Exception as e:
|
||
|
|
logger.warning(place)
|
||
|
|
logger.warning("文档地名检查---组织提出出错",e)
|
||
|
|
continue
|
||
|
|
logger.info(error_places)
|
||
|
|
returnInfo = "发现异常地名<br>"
|
||
|
|
if len(error_places)>0:
|
||
|
|
for t in error_places:
|
||
|
|
keyword= t['placeName'].replace("\n","")
|
||
|
|
# 查找包含关键字的段落
|
||
|
|
paragraphs = re.findall(r'.*?' + re.escape(keyword) + r'.*?\n', gettext)
|
||
|
|
yuanwen= paragraphs[0].replace(keyword,f"**{keyword}**").replace("\n","")
|
||
|
|
returnInfo+="原文：" + yuanwen + "<br>出现异常地名：**" + keyword + "**！请注意" + "<br>"
|
||
|
|
yield returnInfo
|
||
|
|
logger.info(returnInfo)
|
||
|
|
else:
|
||
|
|
yield "**未发现发现异常地名**"
|