# -*- coding:utf-8 -*- import time from docx import Document from paddlenlp import Taskflow from qwen_agent.agents import Assistant import re import json_repair import math from docx.opc.pkgreader import _SerializedRelationships, _SerializedRelationship from docx.opc.oxml import parse_xml def load_from_xml_v2(baseURI, rels_item_xml): """ Return |_SerializedRelationships| instance loaded with the relationships contained in *rels_item_xml*. Returns an empty collection if *rels_item_xml* is |None|. """ srels = _SerializedRelationships() if rels_item_xml is not None: rels_elm = parse_xml(rels_item_xml) for rel_elm in rels_elm.Relationship_lst: if rel_elm.target_ref in ('../NULL', 'NULL'): continue srels._srels.append(_SerializedRelationship(baseURI, rel_elm)) return srels _SerializedRelationships.load_from_xml = load_from_xml_v2 import logging import logging.config log_config = { 'version': 1, 'disable_existing_loggers': False, 'formatters': { 'standard': { 'format': '%(asctime)s - %(name)s - %(levelname)s - %(message)s', }, }, 'handlers': { 'console': { 'class': 'logging.StreamHandler', 'formatter': 'standard', 'level': logging.INFO, }, 'file': { 'class': 'logging.FileHandler', 'filename': 'Logger.log', 'formatter': 'standard', 'level': logging.INFO, }, }, 'loggers': { '': { 'handlers': ['console', 'file'], 'level': logging.INFO, 'propagate': True, }, } } logging.config.dictConfig(log_config) logger = logging.getLogger("checkCompanyName") prompt = ''' .根据上述文本判断,是否为具体的公司或组织名称,你可以使用工具利用互联网查询, 你只能在[具体的公司或组织名称,公益组织,简称,统称,泛化组织,政府单位,机关单位,学校,行业类型,其他]选项中选择答案, 回答格式[{“companyName”:“名称”,"回答":"答案"},{“companyName”:“名称”,"回答":"答案"}],不做过多的解释,严格按回答格式作答; ''' llm_cfg = { #'model': 'qwen1.5-72b-chat', 'model':"qwen2-72b", 'model_server': 'http://127.0.0.1:1025/v1', # base_url, also known as api_base # 'api_key': 'sk-ea89cf04431645b185990b8af8c9bb13', } bot = Assistant(llm=llm_cfg, name='Assistant', # system_message="你是一个地理专家,可以准确的判断地理位置,如果你不确定,可以使用工具" ) def getDocxToTextAll(name): docxPath=name document = Document(docxPath) # 逐段读取docx文档的内容 levelList=[] words=[] addStart = False levelText="" i = 0 for paragraph in document.paragraphs: # 判断该段落的标题级别 # 这里用isTitle()临时代表,具体见下文介绍的方法 text = paragraph.text if text.strip():#非空判断 # print("非空") words.append(text) # 将所有段落文本拼接成一个字符串,并用换行符分隔 text = '\n'.join(words) # 将文本写入txt文件 with open("checkCompanyName.txt", 'w', encoding='utf-8') as txt_file: txt_file.write(text) def companyNameTask(text): yield "文档公司或组织名称检查---启动中...." wordtag = Taskflow("knowledge_mining",device_id=0) batchNum=20 sentences = re.split(r'[。\n]', text) # 去掉空字符 sentences = [sentence.strip() for sentence in sentences if sentence.strip()] # 计算总字符数 total_chars = len(sentences) # 计算有多少份 num_chunks = math.ceil(total_chars / batchNum) # 按batchNum字为一份进行处理 chunks = [sentences[i:i + batchNum] for i in range(0, total_chars, batchNum)] placeList = [] # 打印每一份的内容 for i, chunk in enumerate(chunks): yield f"文档公司或组织名称检查---文档解析进度:{i + 1}/{num_chunks}" wenBen=".".join(chunk) try: res = wordtag(wenBen) except Exception as e: logging.warning(chunk) logging.warning("文档公司或组织名称检查---词类分析出错",e) continue isplace = False for zuhe in res[0]['items']: # 上一个的地名,这一个还是地名,就和上一个相加代替这个 zhi = zuhe.get("wordtag_label") if isplace: name = placeList[len(placeList) - 1] if zhi.find("组织机构类") >= 0: # or zuhe[1] == "ns" isplace = True new_text = zuhe['item'].replace("\n", "") placeList[len(placeList) - 1] = name + new_text continue if zhi.find("组织机构类") >= 0: isplace = True new_text = zuhe['item'].replace("\n", "") placeList.append(new_text) else: isplace = False # 打印总份数 yield "文档公司或组织名称检查---文档解析完成" placeList=list(dict.fromkeys(placeList)) yield placeList def checkCompanyName(filename): yield f"文档公司或组织名称检查---开始处理文档..." try: getDocxToTextAll(filename) except Exception as e: logging.warning(e) yield "文档公司或组织名称检查---文档无法打开,请检查文档内容" return with open("checkCompanyName.txt", "r", encoding='utf-8') as f: gettext = f.read() yield f"文档公司或组织名称检查---开始解析文档..." # 每次生成一个数字就发送 for item in companyNameTask(gettext): if isinstance(item, str): yield item else: final_list = item # 获取最终结果 propnStr = ",".join(final_list) messages = [{'role': 'user', 'content': [{'text': propnStr+prompt}]}] runList = [] yield f"文档公司或组织名称检查---结果生成中..." # 每次生成一个数字就发送 cishu = 0 for rsp in bot.run(messages): runList.append(rsp) if cishu > 3: cishu = 0 yield "文档公司或组织名称检查---结果生成中" + '.' * cishu cishu += 1 data = runList[len(runList) - 1][0]["content"] parsed_data = json_repair.loads(data.replace('`', '')) error_places=[] for place in parsed_data: try: if place['回答'] == '非泛化的公司或组织名称': error_places.append(place) except Exception as e: logging.warning(place) logging.warning("文档公司或组织名称检查---组织提出出错",e) continue logging.info(error_places) returnInfo = "发现异常公司或组织名称
" if len(error_places)>0: for t in error_places: keyword= t['companyName'].replace("\n","") # 查找包含关键字的段落 paragraphs = re.findall(r'.*?' + re.escape(keyword) + r'.*?\n', gettext) t["yuanwen"]=paragraphs[0] yuanwen = paragraphs[0].replace(keyword, f"**{keyword}**").replace("\n","") returnInfo += "原文:" + yuanwen + "
异常公司或组织名称:**" + keyword + "**!请注意" + "
" logging.info(returnInfo) yield returnInfo else: yield "**未发现异常公司或组织名称**
"