python_ai/checkCompanyName.py

# -*- coding:utf-8 -*-
from docx import Document
from qwen_agent.agents import Assistant
import re
import json_repair
import json
import math
from docx.opc.pkgreader import _SerializedRelationships, _SerializedRelationship
from docx.opc.oxml import parse_xml
import requests
# from myLogger import outLog
import time


def load_from_xml_v2(baseURI, rels_item_xml):
    """
    Return |_SerializedRelationships| instance loaded with the
    relationships contained in *rels_item_xml*. Returns an empty
    collection if *rels_item_xml* is |None|.
    """
    srels = _SerializedRelationships()
    if rels_item_xml is not None:
        rels_elm = parse_xml(rels_item_xml)
        for rel_elm in rels_elm.Relationship_lst:
            if rel_elm.target_ref in ('../NULL', 'NULL'):
                continue
            srels._srels.append(_SerializedRelationship(baseURI, rel_elm))
    return srels


_SerializedRelationships.load_from_xml = load_from_xml_v2

import logging

# outLog.logger = logging.getLogger("checkCompanyName")
userLog = None
prompt = '''
.根据上述文本判断，是否为具体的公司或组织名称，你可以使用工具利用互联网查询，
你只能在[具体的公司或组织名称,公益组织,简称,统称,泛化组织,政府单位,机关单位,学校，行业类型，其他]选项中选择答案,
回答格式[{“companyName”：“名称”,"回答":"答案"}，{“companyName”：“名称”,"回答":"答案"}]，不做过多的解释,严格按回答格式作答;
'''
llm_cfg = {
    # 'model': 'qwen1.5-72b-chat',
    'model': "qwen2-72b",
    'model_server': 'http://127.0.0.1:1025/v1',  # base_url, also known as api_base
    # 'api_key': 'sk-ea89cf04431645b185990b8af8c9bb13',
}
bot = Assistant(llm=llm_cfg,
                name='Assistant',
                # system_message="你是一个地理专家，可以准确的判断地理位置，如果你不确定，可以使用工具"
                )


def getDocxToTextAll(name):
    docxPath = name
    loopCount = 0
    while True:
        loopCount += 1
        if (loopCount >= 60):
            raise Exception("文档读取超时，或文档存在问题无法读取")
            break
        try:
            document = Document(docxPath)
            break
        except Exception as e:
            time.sleep(1)
            pass
    # 逐段读取docx文档的内容
    words = []
    i = 0
    for paragraph in document.paragraphs:
        # 判断该段落的标题级别
        # 这里用isTitle()临时代表，具体见下文介绍的方法
        text = paragraph.text
        if text.strip():  # 非空判断
            # print("非空")
            words.append(text)
    # 将所有段落文本拼接成一个字符串，并用换行符分隔
    text = '\n'.join(words)
    # 将文本写入txt文件
    with open("checkCompanyName.txt", 'w', encoding='utf-8') as txt_file:
        txt_file.write(text)


def companyNameTask(text):
    yield "文档公司或组织名称检查---文档解析中...."
    userLog.info("文档公司或组织名称检查---任务开始")
    batchNum = 5
    sentences = re.split(r'[、，。\n]', text)
    # 去掉空字符
    sentences = [sentence.strip() for sentence in sentences if sentence.strip()]
    # 计算总字符数
    total_chars = len(sentences)

    # 计算有多少份
    num_chunks = math.ceil(total_chars / batchNum)

    # 按batchNum字为一份进行处理
    chunks = [sentences[i:i + batchNum] for i in range(0, total_chars, batchNum)]
    placeList = []
    # 打印每一份的内容
    for i, chunk in enumerate(chunks):
        yield f"文档公司或组织名称检查---文档解析进度:{i + 1}/{num_chunks}"
        try:
            # wenBen = ".".join(chunk)
            url = "http://0.0.0.0:8191/taskflow/checkPlaceNameServer"
            headers = {"Content-Type": "application/json"}
            data = {
                "data": {
                    "text": chunk,
                    # "text":wenBen
                }
            }
            r = requests.post(url=url, headers=headers, data=json.dumps(data))
            res = json.loads(r.text)
            res = res["data"]
            # print(res)
        except Exception as e:
            userLog.warning(chunk)
            userLog.warning("文档公司或组织名称检查--错别字识别出错\n")
            userLog.warning(e)
            return
        isplace = False

        # for zuhe in res:
        #     # 上一个的地名,这一个还是地名，就和上一个相加代替这个
        #     if isplace:
        #         name = placeList[len(placeList) - 1]
        #         if zuhe[1].find("组织机构类") >= 0:  # or zuhe[1] == "ns"
        #             isplace = True
        #             new_text = zuhe[0].replace("\n", "")
        #             placeList[len(placeList) - 1] = name + new_text
        #             continue
        #     if zuhe[1].find("组织机构类") >= 0:
        #         isplace = True
        #         new_text = zuhe[0].replace("\n", "")
        #         placeList.append(new_text)
        #     else:
        #         isplace = False
        ##案例[[('目前', 'TIME'), ('江北区历史文化档案馆', 'ORG')], [('宁波国研简直，并且在东软', 'ORG'), ('宁波市北仑区教育局', 'ORG'), ('国研信息', 'ORG'), ('浙江省', 'LOC'), ('宁波市金凤区', 'LOC'), ('金凤区', 'LOC')]]
        for zuhe in res:
            # 上一个的地名,这一个还是地名，就和上一个相加代替这个
            for chid in zuhe:
                if (chid[1] == "ORG"):
                    new_text = chid[0].replace("\n", "")
                    placeList.append(new_text)
    # 打印总份数
    yield "文档公司或组织名称检查---文档解析完成"
    placeList = list(dict.fromkeys(placeList))
    userLog.debug(placeList)
    yield placeList


def checkCompanyName(filename, user_id, outLog):
    yield f"文档公司或组织名称检查---开始处理文档..."
    global userLog
    userLog = outLog.get_queue(user_id, "checkCompanyName")
    try:
        getDocxToTextAll(filename)
    except Exception as e:
        userLog.warning(e)
        userLog.warning("文档公司或组织名称检查---文档无法打开，请检查文档内容")
        yield "文档公司或组织名称检查---文件无法正常打开。可以尝试用WORD或WPS打开文件，进行修复并另存，用另存的文件再做一次尝试。"
        outLog.mark_done(user_id, "checkCompanyName")
        return
    with open("checkCompanyName.txt", "r", encoding='utf-8') as f:
        gettext = f.read()
    yield f"文档公司或组织名称检查---开始解析文档..."  # 每次生成一个数字就发送
    final_list = ""
    for item in companyNameTask(gettext):
        if isinstance(item, str):
            yield item
        else:
            final_list = item  # 获取最终结果
    propnStr = ",".join(final_list)
    messages = [{'role': 'user', 'content': [{'text': propnStr + prompt}]}]
    runList = []
    yield f"文档公司或组织名称检查---结果生成中..."  # 每次生成一个数字就发送
    cishu = 0
    for rsp in bot.run(messages):
        runList.append(rsp)
        if cishu > 3:
            cishu = 0
        yield "文档公司或组织名称检查---结果生成中" + '.' * cishu
        cishu += 1
    data = runList[len(runList) - 1][0]["content"]
    parsed_data = json_repair.loads(data.replace('`', ''))
    error_places = []

    for place in parsed_data:
        try:
            if place['回答'] == '具体的公司或组织名称':
                if (place["companyName"] == "北京国研科技咨询有限公司浙江分公司"):
                    continue
                error_places.append(place)
        except Exception as e:
            userLog.warning(place)
            userLog.warning(e)
            userLog.warning("文档公司或组织名称检查---组织提出出错")
            continue
    returnInfo = "发现异常公司或组织名称<br>"
    if len(error_places) > 0:
        for t in error_places:
            keyword = t['companyName'].replace("\n", "")
            # 查找包含关键字的段落
            paragraphs = re.findall(r'.*?' + re.escape(keyword) + r'.*?\n', gettext)
            t["yuanwen"] = paragraphs[0]
            yuanwen = paragraphs[0].replace(keyword, f"**{keyword}**").replace("\n", "")
            returnInfo += "原文：" + yuanwen + "<br>异常公司或组织名称：**" + keyword + "**！请注意" + "<br>"
            userLog.info("文档公司或组织名称检查---原文：" + yuanwen + "异常公司或组织名称：" + keyword + "！请注意")
        yield returnInfo
    else:
        yield "**未发现异常公司或组织名称**<br>"
        userLog.info("文档公司或组织名称检查---未发现异常公司或组织名称")
    outLog.mark_done(user_id, "checkCompanyName")