You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
221 lines
9.0 KiB
221 lines
9.0 KiB
# -*- coding:utf-8 -*-
|
|
from qwen_agent.agents import Assistant
|
|
from docx import Document
|
|
import re
|
|
import json
|
|
import json_repair
|
|
import math
|
|
from docx.opc.pkgreader import _SerializedRelationships, _SerializedRelationship
|
|
from docx.opc.oxml import parse_xml
|
|
import requests
|
|
# from myLogger import outLog
|
|
import time
|
|
def load_from_xml_v2(baseURI, rels_item_xml):
|
|
"""
|
|
Return |_SerializedRelationships| instance loaded with the
|
|
relationships contained in *rels_item_xml*. Returns an empty
|
|
collection if *rels_item_xml* is |None|.
|
|
"""
|
|
srels = _SerializedRelationships()
|
|
if rels_item_xml is not None:
|
|
rels_elm = parse_xml(rels_item_xml)
|
|
for rel_elm in rels_elm.Relationship_lst:
|
|
if rel_elm.target_ref in ('../NULL', 'NULL'):
|
|
continue
|
|
srels._srels.append(_SerializedRelationship(baseURI, rel_elm))
|
|
return srels
|
|
|
|
|
|
_SerializedRelationships.load_from_xml = load_from_xml_v2
|
|
# import logging
|
|
|
|
# outLog.logger = logging.getLogger("checkDocumentError")
|
|
userLog=None
|
|
llm_cfg = {
|
|
# 'model': 'qwen1.5-72b-chat',
|
|
'model': "qwen2-72b",
|
|
'model_server': 'http://127.0.0.1:1025/v1', # base_url, also known as api_base
|
|
# 'api_key': 'sk-ea89cf04431645b185990b8af8c9bb13',
|
|
}
|
|
bot = Assistant(llm=llm_cfg,
|
|
name='Assistant',
|
|
# description='使用RAG检索并回答，支持文件类型：PDF/Word/PPT/TXT/HTML。'
|
|
system_message="你是一个错别字分析大师"
|
|
)
|
|
# prompt='''
|
|
# 是否存在错别字，若存在请指出，不做其他方面的校验，你只能在[存在，不存在，未知]选项中选择答案,
|
|
# 回答格式[{“placeName”：“原文”,"改正后":"改正的内容","回答":"答案"},{“placeName”：“原文”,"改正后":"改正的内容","回答":"答案"}]，不做过多的解释,严格按回答格式作答;
|
|
# '''
|
|
prompt = '''
|
|
请回答以上问题，[是，否]选项中选择答案,原文内容，标点符号保持不变，如果有错请给出详细的解析，没有错则不用给解析
|
|
回答格式请按照以下json格式[{"placeName":"序号值","回答":"答案","解析","解析内容"},{"placeName":"序号值","回答":"答案","解析","解析内容"}]，不做过多的解释,严格按回答格式作答;
|
|
'''
|
|
|
|
|
|
def getDocxToTextAll(name):
|
|
docxPath = name
|
|
loopCount = 0
|
|
document = Document(docxPath)
|
|
# while True:
|
|
# loopCount+=1
|
|
# if(loopCount>=60):
|
|
# raise Exception("文档读取超时，或文档存在问题无法读取")
|
|
# break
|
|
# try:
|
|
# document = Document(docxPath)
|
|
# break
|
|
# except Exception as e:
|
|
# time.sleep(1)
|
|
# pass
|
|
# 逐段读取docx文档的内容
|
|
words = []
|
|
for paragraph in document.paragraphs:
|
|
# 判断该段落的标题级别
|
|
# 这里用isTitle()临时代表，具体见下文介绍的方法
|
|
text = paragraph.text
|
|
if text.strip(): # 非空判断
|
|
# print("非空")
|
|
words.append(text)
|
|
# 将所有段落文本拼接成一个字符串，并用换行符分隔
|
|
text = '\n'.join(words)
|
|
|
|
# 将文本写入txt文件
|
|
with open("checkDocumentError.txt", 'w', encoding='utf-8') as txt_file:
|
|
txt_file.write(text)
|
|
|
|
|
|
def checkDocumentError(filename,user_id,outLog):
|
|
global userLog
|
|
userLog=outLog.get_queue(user_id,"checkDocumentError")
|
|
yield f"文档纠错---开始处理文档..."
|
|
try:
|
|
getDocxToTextAll(filename)
|
|
except Exception as e:
|
|
userLog.warning(e)
|
|
userLog.warning("文档纠错----文档无法打开，请检查文档内容")
|
|
yield "文档纠错----文件无法正常打开。可以尝试用WORD或WPS打开文件，进行修复并另存，用另存的文件再做一次尝试。"
|
|
outLog.mark_done(user_id, "checkDocumentError")
|
|
return
|
|
with open("checkDocumentError.txt", "r", encoding='utf-8') as f:
|
|
gettext = f.read()
|
|
yield f"文档纠错---开始解析文档..." # 每次生成一个数字就发送
|
|
final_list = []
|
|
for item in documentErrorTask(gettext):
|
|
if isinstance(item, str):
|
|
yield item
|
|
else:
|
|
final_list = item # 获取最终结果
|
|
resInfo = "发现错别字<br>"
|
|
if (len(final_list) > 0):
|
|
for i in final_list:
|
|
yuanwen = i["placeName"].replace("\n", "")
|
|
jianyi = i["jianyi"].replace("\n", "")
|
|
resInfo += "原文：" + yuanwen + "<br>建议：**" + jianyi + "**<br>"
|
|
yield resInfo
|
|
|
|
else:
|
|
yield "**未发现错别字**"
|
|
userLog.info("文档纠错---未发现错别字")
|
|
outLog.mark_done(user_id,"checkDocumentError")
|
|
|
|
|
|
def documentErrorTask(text):
|
|
"""
|
|
分批读取文本文件
|
|
:param file_path: 文件路径
|
|
:param batch_size: 每批处理的字符数
|
|
:return: 生成器，每次返回一批文本
|
|
"""
|
|
yield "文档纠错---文档解析中...."
|
|
userLog.info("文档纠错---任务开始")
|
|
batchNum = 20
|
|
sentences = re.split(r'[。\n]', text)
|
|
# 去掉空字符
|
|
sentences = [sentence.strip() for sentence in sentences if sentence.strip()]
|
|
# 计算总字符数
|
|
total_chars = len(sentences)
|
|
# 计算有多少份
|
|
num_chunks = math.ceil(total_chars / batchNum)
|
|
# 按batchNum字为一份进行处理
|
|
chunks = [sentences[i:i + batchNum] for i in range(0, total_chars, batchNum)]
|
|
# 打印每一份的内容
|
|
err = []
|
|
for i, chunk in enumerate(chunks):
|
|
yield f"文档纠错---文档解析进度:{i + 1}/{num_chunks}"
|
|
try:
|
|
# url = "http://0.0.0.0:8190/taskflow/checkDocumentError"
|
|
# headers = {"Content-Type": "application/json"}
|
|
# data = {
|
|
# "data": {
|
|
# "text": chunk,
|
|
# }
|
|
# }
|
|
# r = requests.post(url=url, headers=headers, data=json.dumps(data))
|
|
# res = json.loads(r.text)
|
|
url = "http://127.0.0.1:5001/taskflow/checkDocumentError"
|
|
headers = {"Content-Type": "application/json"}
|
|
data = {
|
|
"data": {
|
|
"text": chunk,
|
|
}
|
|
}
|
|
r = requests.post(url=url, headers=headers, data=json.dumps(data))
|
|
res = json.loads(r.text)
|
|
except Exception as e:
|
|
userLog.warning(chunk)
|
|
userLog.warning("文档纠错--错别字识别出错\n")
|
|
userLog.warning(e)
|
|
continue
|
|
lines_with_greeting = [place for place in res["data"] if len(place['errors']) > 0]
|
|
userLog.debug(lines_with_greeting)
|
|
if len(lines_with_greeting) > 0:
|
|
num = 0
|
|
wenti = [] # 记录问题的数组
|
|
keyword_list = [] # 记录问题
|
|
for t in lines_with_greeting:
|
|
temp_errorWords = []
|
|
keyword = t['source']
|
|
keyword_list.append(keyword)
|
|
for item in t["errors"]:
|
|
# for key, value in item['correction'].items():
|
|
# temp_errorWords.append(key)
|
|
temp_errorWords.append(item[0])
|
|
wenti.append(
|
|
# "{}：原文是{}。问题：【{}】这些字是否为当前原文的错别字".format(num, keyword, ",".join(temp_errorWords)))
|
|
"{}：原文是{}。问题：当前原文是否存在错别字,只检查错被子，其他不做分析".format(num, keyword))
|
|
num += 1
|
|
words = "\n".join(wenti)
|
|
userLog.debug(words)
|
|
messages = [{'role': 'user', 'content': [{'text': words + prompt}]}]
|
|
runList = []
|
|
yield f"文档纠错---内容解析中..." # 每次生成一个数字就发送
|
|
cishu = 0
|
|
for rsp in bot.run(messages):
|
|
runList.append(rsp)
|
|
if cishu > 3:
|
|
cishu = 0
|
|
yield "文档纠错---内容解析中" + '.' * cishu
|
|
cishu += 1
|
|
data = runList[len(runList) - 1][0]["content"]
|
|
parsed_data = json_repair.loads(data.replace("\\", "").replace('`', ''))
|
|
userLog.debug(parsed_data)
|
|
resListerr = []
|
|
for place in parsed_data:
|
|
try:
|
|
if place['回答'] == '是':
|
|
place["placeName"] = keyword_list[int(place["placeName"])]
|
|
place["jianyi"] = place["解析"]
|
|
resListerr.append(place)
|
|
userLog.info("文档纠错---原文：" + place["placeName"] + "<br>建议：" + place["jianyi"])
|
|
except Exception as e:
|
|
userLog.warning(parsed_data)
|
|
userLog.warning(place)
|
|
userLog.warning("文档纠错--错别字提取出错\n")
|
|
userLog.warning(e)
|
|
continue
|
|
if (len(resListerr) > 0):
|
|
err.extend(resListerr)
|
|
# 打印总份数
|
|
yield "文档纠错---文档解析完成"
|
|
userLog.info("文档纠错---任务结束")
|
|
yield err
|