zhouhaibin
4 months ago
3 changed files with 346 additions and 0 deletions
@ -0,0 +1,52 @@ |
|||||
|
# from paddlenlp import Taskflow |
||||
|
#checkPlaceNameServer |
||||
|
# tagTask1 = Taskflow("ner",device_id=2,precision='fp16')##checkPlaceName |
||||
|
# from flask import Flask, request, jsonify |
||||
|
from fastapi import FastAPI, Request |
||||
|
from transformers import AutoTokenizer, AutoModelForCausalLM, GenerationConfig |
||||
|
import uvicorn |
||||
|
from fastapi.responses import JSONResponse |
||||
|
from pydantic import BaseModel |
||||
|
app = FastAPI() |
||||
|
|
||||
|
import sys |
||||
|
sys.path.append("..") |
||||
|
from ner_model import NERModel |
||||
|
# model = NERModel("bert", "shibing624/bert4ner-base-chinese") |
||||
|
model = NERModel("bertspan", "shibing624/bertspan4ner-base-chinese",cuda_device=5) |
||||
|
class RequestData(BaseModel): |
||||
|
data: dict |
||||
|
@app.post("/taskflow/checkPlaceNameServer") |
||||
|
async def process_request(request: RequestData): |
||||
|
global model |
||||
|
# 提取文本数据 |
||||
|
text_data = request.data.get('text') |
||||
|
# 处理文本数据，例如检查错误等 |
||||
|
# 这里您可以添加实际的逻辑来检查文档错误 |
||||
|
predictions, raw_outputs, entities = model.predict(text_data) |
||||
|
|
||||
|
# 返回响应 |
||||
|
return JSONResponse(content={"status": "success", "data": entities}, status_code=200) |
||||
|
if __name__ == "__main__": |
||||
|
uvicorn.run(app, host="0.0.0.0", port=8191) |
||||
|
# # 创建一个锁对象 |
||||
|
# import threading |
||||
|
# app = Flask(__name__) |
||||
|
# lock = threading.Lock() |
||||
|
# #多线程但是每次只处理一个请求，多余的请求需要排队 |
||||
|
# @app.route('/taskflow/checkPlaceNameServer', methods=['POST']) |
||||
|
# def process_request(): |
||||
|
# with lock: |
||||
|
# data = request.get_json() |
||||
|
# # print("data",data) |
||||
|
# # 提取文本数据 |
||||
|
# text_data = data.get('data', {}).get('text') |
||||
|
# # 处理文本数据，例如检查错误等 |
||||
|
# # 这里您可以添加实际的逻辑来检查文档错误 |
||||
|
# predictions, raw_outputs, entities =model.predict(text_data) |
||||
|
# # 示例：简单打印接收到的文本 |
||||
|
# # # 返回响应 |
||||
|
# return jsonify({"status": "success", "data": entities}), 200 |
||||
|
|
||||
|
# if __name__ == '__main__': |
||||
|
# app.run(threaded=True,port=8191) |
||||
@ -0,0 +1,71 @@ |
|||||
|
# from paddlenlp import Taskflow |
||||
|
# similarity1 = Taskflow("text_similarity",device_id=3,precision='fp16')##checkRepeatText |
||||
|
# from flask import Flask, request, jsonify |
||||
|
# import threading |
||||
|
# app = Flask(__name__) |
||||
|
|
||||
|
# # 创建一个锁对象 |
||||
|
# lock = threading.Lock() |
||||
|
# @app.route('/taskflow/checkRepeatText', methods=['POST']) |
||||
|
# def process_request(): |
||||
|
# with lock: |
||||
|
# data = request.get_json() |
||||
|
# # print("data",data) |
||||
|
# # 提取文本数据 |
||||
|
# text_data = data.get('data', {}).get('text') |
||||
|
# # 处理文本数据，例如检查错误等 |
||||
|
# # 这里您可以添加实际的逻辑来检查文档错误 |
||||
|
# res =similarity1(text_data) |
||||
|
# # 示例：简单打印接收到的文本 |
||||
|
# # # 返回响应 |
||||
|
# return jsonify({"status": "success", "data": res}), 200 |
||||
|
|
||||
|
# if __name__ == '__main__': |
||||
|
# app.run(threaded=True,port=8192) |
||||
|
from sentence_transformers import SentenceTransformer, util |
||||
|
import itertools |
||||
|
from fastapi import FastAPI, Request |
||||
|
from transformers import AutoTokenizer, AutoModelForCausalLM, GenerationConfig |
||||
|
import uvicorn |
||||
|
from fastapi.responses import JSONResponse |
||||
|
from pydantic import BaseModel |
||||
|
import torch |
||||
|
app = FastAPI() |
||||
|
model = SentenceTransformer("shibing624/text2vec-base-chinese",device="npu:5") |
||||
|
|
||||
|
class RequestData(BaseModel): |
||||
|
data: dict |
||||
|
@app.post("/taskflow/checkRepeatText") |
||||
|
async def process_request(request: RequestData): |
||||
|
global model |
||||
|
# 提取文本数据 |
||||
|
text_data = request.data.get('text') |
||||
|
a=text_data[0][0] |
||||
|
b=text_data[0][1] |
||||
|
emb_a = model.encode(a) |
||||
|
emb_b = model.encode(b) |
||||
|
cos_sim = util.cos_sim(emb_a, emb_b) |
||||
|
results = [] |
||||
|
results.append({"text1":a,"text2":b,"similarity":cos_sim.item()}) |
||||
|
# 返回响应 |
||||
|
return JSONResponse(content={"status": "success", "data": results}, status_code=200) |
||||
|
@app.post("/taskflow/getRepeatText") |
||||
|
async def process_request(request: RequestData): |
||||
|
global model |
||||
|
# 提取文本数据 |
||||
|
text_data = request.data.get('text') |
||||
|
allcorpus =text_data[0] #全部文档信息 |
||||
|
query=text_data[1] #要查询的文档信息 |
||||
|
corpus_embeddings = model.encode(allcorpus, convert_to_tensor=True) |
||||
|
top_k = min(4, len(allcorpus)) |
||||
|
query_embedding = model.encode(query, convert_to_tensor=True) |
||||
|
cos_scores = util.cos_sim(query_embedding, corpus_embeddings)[0] |
||||
|
top_results = torch.topk(cos_scores, k=top_k) |
||||
|
results = [] |
||||
|
for score, idx in zip(top_results[0], top_results[1]): |
||||
|
print(allcorpus[idx], "(Score: {:.4f})".format(score.item())) |
||||
|
results.append({"text1":allcorpus[idx],"text2":query,"similarity":score.item()}) |
||||
|
# 返回响应 |
||||
|
return JSONResponse(content={"status": "success", "data": results}, status_code=200) |
||||
|
if __name__ == "__main__": |
||||
|
uvicorn.run(app, host="0.0.0.0", port=8192) |
||||
@ -0,0 +1,223 @@ |
|||||
|
import threading |
||||
|
# from checkPlaceName import checkPlaceName |
||||
|
# from checkRepeatText import checkRepeatText |
||||
|
# from checkCompanyName import checkCompanyName |
||||
|
# from checkDocumentError import checkDocumentError |
||||
|
# from checkTitleName import checkTitleName |
||||
|
# from myLogger import outLog |
||||
|
# import time |
||||
|
# def run_check_company_name(filename,user_id): |
||||
|
# for i in checkCompanyName(filename,user_id): |
||||
|
# pass |
||||
|
|
||||
|
# def run_get_document_error(filename,user_id): |
||||
|
# for i in checkDocumentError(filename,user_id): |
||||
|
# pass |
||||
|
# def runcheckTitleName(filename,user_id): |
||||
|
# for i in checkTitleName(filename,user_id): |
||||
|
# pass |
||||
|
# def runcheckRepeatText(filename,user_id): |
||||
|
# for i in checkRepeatText(filename,user_id): |
||||
|
# pass |
||||
|
# def runcheckPlaceName(filename,user_id): |
||||
|
# for i in checkPlaceName(filename,user_id): |
||||
|
# pass |
||||
|
|
||||
|
# def get(user_id): |
||||
|
# time.sleep(5) |
||||
|
# while True: |
||||
|
# if outLog.is_done(user_id): |
||||
|
# break |
||||
|
# q = outLog.get_queueData(user_id) |
||||
|
# if q: |
||||
|
# text = q.pop(0) |
||||
|
# print(text) |
||||
|
# print("打印结束") |
||||
|
|
||||
|
# filename = "17.docx" |
||||
|
|
||||
|
# # 创建线程getapp |
||||
|
# thread1 = threading.Thread(target=run_check_company_name, args=(filename,"1")) |
||||
|
# thread2 = threading.Thread(target=run_get_document_error, args=(filename,"1")) |
||||
|
# thread3 = threading.Thread(target=runcheckTitleName, args=(filename,"1")) |
||||
|
# thread4 = threading.Thread(target=runcheckRepeatText, args=(filename,"1")) |
||||
|
# thread5 = threading.Thread(target=runcheckPlaceName, args=(filename,"1")) |
||||
|
# thread6 = threading.Thread(target=get, args=("1",)) |
||||
|
# thread1 = threading.Thread(target=getapp, args=(filename,)) |
||||
|
# thread2 = threading.Thread(target=getapp, args=(filename,)) |
||||
|
# thread3 = threading.Thread(target=getapp, args=(filename,)) |
||||
|
# thread4 = threading.Thread(target=getapp, args=(filename,)) |
||||
|
# thread5 = threading.Thread(target=getapp, args=(filename,)) |
||||
|
# thread6 = threading.Thread(target=getapp, args=("1",)) |
||||
|
# # 启动线程 |
||||
|
# thread1.start() |
||||
|
# thread2.start() |
||||
|
# thread3.start() |
||||
|
# thread4.start() |
||||
|
# thread5.start() |
||||
|
# thread6.start() |
||||
|
# # 等待线程完成 |
||||
|
# thread1.join() |
||||
|
# thread2.join() |
||||
|
# thread3.join() |
||||
|
# thread4.join() |
||||
|
# thread5.join() |
||||
|
# thread6.join() |
||||
|
# print("Both tasks completed.") |
||||
|
# from pycorrector.macbert.macbert_corrector import MacBertCorrector |
||||
|
|
||||
|
# m = MacBertCorrector("models") |
||||
|
# for i in range(10): |
||||
|
# i = m.correct("行政捡查是行政机关覆行政府职能、管理经济社会事务的重要方式，开展计划统筹是行政检查控总量、提质效的重要措施和手段，直接影响改革或得感和社会满意度") |
||||
|
# print(i) |
||||
|
# import re |
||||
|
# import json |
||||
|
# import json_repair |
||||
|
# import math |
||||
|
# import os |
||||
|
# import platform |
||||
|
# import torch |
||||
|
# import torch_npu |
||||
|
# import operator |
||||
|
# from torch_npu.contrib import transfer_to_npu |
||||
|
# torch_device = "npu:4" # 0~7 |
||||
|
# torch.npu.set_device(torch.device(torch_device)) |
||||
|
# torch.npu.set_compile_mode(jit_compile=False) |
||||
|
# from transformers import BertTokenizerFast,BertForMaskedLM |
||||
|
# # option = {} |
||||
|
# # option["NPU_FUZZY_COMPILE_BLACKLIST"] = "Tril" |
||||
|
# # torch.npu.set_option(option) |
||||
|
# print("torch && torch_npu import successfully") |
||||
|
|
||||
|
# DEFAULT_CKPT_PATH = 'macbert4csc' |
||||
|
# #models=----macbert4csc-base-chinese |
||||
|
# model = BertForMaskedLM.from_pretrained( |
||||
|
# DEFAULT_CKPT_PATH, |
||||
|
# torch_dtype=torch.float16, |
||||
|
# device_map=torch_device |
||||
|
# ).npu().eval() |
||||
|
# tokenizer = BertTokenizerFast.from_pretrained(DEFAULT_CKPT_PATH) |
||||
|
# def get_errors(corrected_text, origin_text): |
||||
|
# sub_details = [] |
||||
|
# for i, ori_char in enumerate(origin_text): |
||||
|
# if ori_char in [' ', '“', '”', '‘', '’', '琊', '\n', '…', '—', '擤']: |
||||
|
# # add unk word |
||||
|
# corrected_text = corrected_text[:i] + ori_char + corrected_text[i:] |
||||
|
# continue |
||||
|
# if i >= len(corrected_text): |
||||
|
# continue |
||||
|
# if ori_char != corrected_text[i]: |
||||
|
# if ori_char.lower() == corrected_text[i]: |
||||
|
# # pass english upper char |
||||
|
# corrected_text = corrected_text[:i] + ori_char + corrected_text[i + 1:] |
||||
|
# continue |
||||
|
# sub_details.append((ori_char, corrected_text[i], i, i + 1)) |
||||
|
# sub_details = sorted(sub_details, key=operator.itemgetter(2)) |
||||
|
# return corrected_text, sub_details |
||||
|
# result = [] |
||||
|
# def getapp(gettext): |
||||
|
# result = [] |
||||
|
# batchNum = 20 |
||||
|
# sentences = re.split(r'[。\n]', gettext) |
||||
|
# # 去掉空字符 |
||||
|
# sentences = [sentence.strip() for sentence in sentences if sentence.strip()] |
||||
|
# # 计算总字符数 |
||||
|
# total_chars = len(sentences) |
||||
|
|
||||
|
# # 计算有多少份 |
||||
|
# num_chunks = math.ceil(total_chars / batchNum) |
||||
|
|
||||
|
# # 按batchNum字为一份进行处理 |
||||
|
# chunks = [sentences[i:i + batchNum] for i in range(0, total_chars, batchNum)] |
||||
|
# # 打印每一份的内容 |
||||
|
# err = [] |
||||
|
# for i, chunk in enumerate(chunks): |
||||
|
# inputs = tokenizer(chunk, padding=True, return_tensors='pt').to(torch_device) |
||||
|
# with torch.no_grad(): |
||||
|
# outputs = model(**inputs) |
||||
|
# for id, (logit_tensor, sentence) in enumerate(zip(outputs.logits, chunk)): |
||||
|
# decode_tokens_new = tokenizer.decode( |
||||
|
# torch.argmax(logit_tensor, dim=-1), skip_special_tokens=True).split(' ') |
||||
|
# decode_tokens_new = decode_tokens_new[:len(sentence)] |
||||
|
# if len(decode_tokens_new) == len(sentence): |
||||
|
# probs = torch.max(torch.softmax(logit_tensor, dim=-1), dim=-1)[0].cpu().numpy() |
||||
|
# decode_str = '' |
||||
|
# for i in range(len(sentence)): |
||||
|
# if probs[i + 1] >= 0.7: |
||||
|
# decode_str += decode_tokens_new[i] |
||||
|
# else: |
||||
|
# decode_str += sentence[i] |
||||
|
# corrected_text = decode_str |
||||
|
# else: |
||||
|
# corrected_text = sentence |
||||
|
# print(corrected_text) |
||||
|
#outputs = model(**tokenizer(chunk, padding=True, return_tensors='pt').to(torch_device)) |
||||
|
# for ids, text in zip(outputs.logits, chunk): |
||||
|
# _text = tokenizer.decode(torch.argmax(ids, dim=-1), skip_special_tokens=True).replace(' ', '') |
||||
|
# corrected_text = _text[:len(text)] |
||||
|
# corrected_text, details = get_errors(corrected_text, text) |
||||
|
# print(text, ' => ', corrected_text, details) |
||||
|
# result.append((corrected_text, details)) |
||||
|
# for i, sent in enumerate(chunk): |
||||
|
# decode_tokens = tokenizer.decode(outputs[i], skip_special_tokens=True).replace(' ', '') |
||||
|
# corrected_sent = decode_tokens[:len(sent)] |
||||
|
# print(corrected_sent) |
||||
|
# corrected_sents.append(corrected_sent) |
||||
|
# from flask import Flask, request, jsonify |
||||
|
# import threading |
||||
|
# import time |
||||
|
# import re |
||||
|
# import math |
||||
|
# from macbert_corrector import MacBertCorrector |
||||
|
# m = MacBertCorrector("macbert4csc") |
||||
|
# app = Flask(__name__) |
||||
|
|
||||
|
# # 创建一个锁对象 |
||||
|
# lock = threading.Lock() |
||||
|
# #多线程但是每次只处理一个请求，多余的请求需要排队 |
||||
|
# @app.route('/taskflow/checkDocumentError', methods=['POST']) |
||||
|
# def process_request(): |
||||
|
# with lock: |
||||
|
|
||||
|
# data = request.get_json() |
||||
|
# # print("data",data) |
||||
|
# # 提取文本数据 |
||||
|
# text_data = data.get('data', {}).get('text', []) |
||||
|
# # print(text_data) |
||||
|
# # 处理文本数据，例如检查错误等 |
||||
|
# # 这里您可以添加实际的逻辑来检查文档错误 |
||||
|
# res = m.correct_batch(text_data) |
||||
|
# # 示例：简单打印接收到的文本 |
||||
|
# # # 返回响应 |
||||
|
# return jsonify({"status": "success", "data": res}), 200 |
||||
|
|
||||
|
# if __name__ == '__main__': |
||||
|
# app.run(threaded=True,port=5001) |
||||
|
from fastapi import FastAPI, Request |
||||
|
from transformers import AutoTokenizer, AutoModelForCausalLM, GenerationConfig |
||||
|
import uvicorn |
||||
|
from fastapi.responses import JSONResponse |
||||
|
from pydantic import BaseModel |
||||
|
app = FastAPI() |
||||
|
from macbert_corrector import MacBertCorrector |
||||
|
m = MacBertCorrector("macbert4csc") |
||||
|
|
||||
|
class RequestData(BaseModel): |
||||
|
data: dict |
||||
|
@app.post("/taskflow/checkDocumentError") |
||||
|
async def process_request(request: RequestData): |
||||
|
global m |
||||
|
# 提取文本数据 |
||||
|
text_data = request.data.get('text') |
||||
|
# 处理文本数据，例如检查错误等 |
||||
|
# 这里您可以添加实际的逻辑来检查文档错误 |
||||
|
|
||||
|
# print(text_data) |
||||
|
# 处理文本数据，例如检查错误等 |
||||
|
# 这里您可以添加实际的逻辑来检查文档错误 |
||||
|
res = m.correct_batch(text_data) |
||||
|
|
||||
|
# 返回响应 |
||||
|
return JSONResponse(content={"status": "success", "data": res}, status_code=200) |
||||
|
if __name__ == "__main__": |
||||
|
uvicorn.run(app, host="0.0.0.0", port=5001) |
||||
Loading…
Reference in new issue