chen jinqian
6 months ago
commit
1fb57e6877
35 changed files with 4828 additions and 0 deletions
@ -0,0 +1,3 @@ |
|||
# Default ignored files |
|||
/shelf/ |
|||
/workspace.xml |
|||
@ -0,0 +1,8 @@ |
|||
<project version="4"> |
|||
<component name="Black"> |
|||
<option name="sdkName" value="Python 3.9 (PyGuoyan)" /> |
|||
</component> |
|||
<component name="ProjectRootManager"> |
|||
<output url="file://$PROJECT_DIR$/out" /> |
|||
</component> |
|||
</project> |
|||
@ -0,0 +1,8 @@ |
|||
<?xml version="1.0" encoding="UTF-8"?> |
|||
<project version="4"> |
|||
<component name="ProjectModuleManager"> |
|||
<modules> |
|||
<module fileurl="file://$PROJECT_DIR$/PyGuoyan.iml" filepath="$PROJECT_DIR$/PyGuoyan.iml" /> |
|||
</modules> |
|||
</component> |
|||
</project> |
|||
@ -0,0 +1,6 @@ |
|||
<?xml version="1.0" encoding="UTF-8"?> |
|||
<project version="4"> |
|||
<component name="PersistentConfig"> |
|||
<option name="langCode" value="en" /> |
|||
</component> |
|||
</project> |
|||
@ -0,0 +1,6 @@ |
|||
<?xml version="1.0" encoding="UTF-8"?> |
|||
<project version="4"> |
|||
<component name="VcsDirectoryMappings"> |
|||
<mapping directory="$PROJECT_DIR$" vcs="Git" /> |
|||
</component> |
|||
</project> |
|||
@ -0,0 +1,9 @@ |
|||
<?xml version="1.0" encoding="UTF-8"?> |
|||
<module type="PYTHON_MODULE" version="4"> |
|||
<component name="NewModuleRootManager" inherit-compiler-output="true"> |
|||
<exclude-output /> |
|||
<content url="file://$MODULE_DIR$" /> |
|||
<orderEntry type="jdk" jdkName="Python 3.9 (PyGuoyan)" jdkType="Python SDK" /> |
|||
<orderEntry type="sourceFolder" forTests="false" /> |
|||
</component> |
|||
</module> |
|||
@ -0,0 +1,14 @@ |
|||
#!/usr/bin/python3 |
|||
from splash.gysplash import SBase |
|||
import json |
|||
|
|||
class SYgcg(SBase): |
|||
def open(self): |
|||
return super().open('ygcg', pages=2, annoucement_type='政府采购') |
|||
|
|||
if __name__ == '__main__': |
|||
test = SYgcg() |
|||
r = test.open() |
|||
|
|||
results = json.loads(r.text) |
|||
print(results) |
|||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -0,0 +1,21 @@ |
|||
-----BEGIN CERTIFICATE----- |
|||
MIIDdTCCAl2gAwIBAgILBAAAAAABFUtaw5QwDQYJKoZIhvcNAQEFBQAwVzELMAkG |
|||
A1UEBhMCQkUxGTAXBgNVBAoTEEdsb2JhbFNpZ24gbnYtc2ExEDAOBgNVBAsTB1Jv |
|||
b3QgQ0ExGzAZBgNVBAMTEkdsb2JhbFNpZ24gUm9vdCBDQTAeFw05ODA5MDExMjAw |
|||
MDBaFw0yODAxMjgxMjAwMDBaMFcxCzAJBgNVBAYTAkJFMRkwFwYDVQQKExBHbG9i |
|||
YWxTaWduIG52LXNhMRAwDgYDVQQLEwdSb290IENBMRswGQYDVQQDExJHbG9iYWxT |
|||
aWduIFJvb3QgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDaDuaZ |
|||
jc6j40+Kfvvxi4Mla+pIH/EqsLmVEQS98GPR4mdmzxzdzxtIK+6NiY6arymAZavp |
|||
xy0Sy6scTHAHoT0KMM0VjU/43dSMUBUc71DuxC73/OlS8pF94G3VNTCOXkNz8kHp |
|||
1Wrjsok6Vjk4bwY8iGlbKk3Fp1S4bInMm/k8yuX9ifUSPJJ4ltbcdG6TRGHRjcdG |
|||
snUOhugZitVtbNV4FpWi6cgKOOvyJBNPc1STE4U6G7weNLWLBYy5d4ux2x8gkasJ |
|||
U26Qzns3dLlwR5EiUWMWea6xrkEmCMgZK9FGqkjWZCrXgzT/LCrBbBlDSgeF59N8 |
|||
9iFo7+ryUp9/k5DPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8E |
|||
BTADAQH/MB0GA1UdDgQWBBRge2YaRQ2XyolQL30EzTSo//z9SzANBgkqhkiG9w0B |
|||
AQUFAAOCAQEA1nPnfE920I2/7LqivjTFKDK1fPxsnCwrvQmeU79rXqoRSLblCKOz |
|||
yj1hTdNGCbM+w6DjY1Ub8rrvrTnhQ7k4o+YviiY776BQVvnGCv04zcQLcFGUl5gE |
|||
38NflNUVyRRBnMRddWQVDf9VMOyGj/8N7yy5Y0b2qvzfvGn9LhJIZJrglfCm7ymP |
|||
AbEVtQwdpf5pLGkkeB6zpxxxYu7KyJesF12KwvhHhm4qxFYxldBniYUr+WymXUad |
|||
DKqC5JlR3XC321Y9YeRq4VzW9v493kHMB65jUr9TU/Qr6cf9tveCX4XSQRjbgbME |
|||
HMUfpIBvFSDJ3gyICh3WZlXi/EjJKSZp4A== |
|||
-----END CERTIFICATE----- |
|||
@ -0,0 +1,5 @@ |
|||
[database] |
|||
host = localhost |
|||
database = guoyantest |
|||
user = root |
|||
password = Guoyan83086775 |
|||
@ -0,0 +1,714 @@ |
|||
#!/usr/bin/python3 |
|||
""" |
|||
=========================================================================================== |
|||
这是一个用于爬取采购信息的模块 |
|||
要处理采购公告信息。主要涉及sc_cggg, calalog, catalogdata, readlog四张表 |
|||
=========================================================================================== |
|||
class Crawler: |
|||
def __init__(self, connect): |
|||
def generate_id(self): |
|||
def write_log_information(self, data_id, catalog_name): |
|||
def CrawlPage_gzw_ningbo(self, page): # 宁波国资委市属国企招标投标信息 |
|||
def CrawlPage_zjcs_nbxzfw(self, type, page): # 宁波市中介超市 |
|||
def CrawlPage_ygcg_nbcqjy_org(self, page): # 宁波市阳光采购 |
|||
def CrawlPage_zfcg_czt_zj(self, page): # 浙江政府采购网 |
|||
def CrawlPage_cbbidding(self, page): # 宁波中基国际招标有限公司 |
|||
def CrawlPage_zmeetb(self, page): # 浙江国际招标有限公司 |
|||
def CrawlPage_nbbidding(self, page): # 宁波国际招标有限公司 |
|||
============================================================================================ |
|||
""" |
|||
|
|||
import datetime |
|||
import hashlib |
|||
import pymysql |
|||
import json |
|||
import random |
|||
from requests_html import HTMLSession |
|||
from requests_html import HTML, UserAgent |
|||
import gymailer |
|||
import time |
|||
|
|||
''' |
|||
============================================================ |
|||
这个类用来封装splash服务 |
|||
其中: |
|||
self.splash_ip 参数是splash服务的ip |
|||
============================================================ |
|||
''' |
|||
|
|||
class Splash: |
|||
def __init__(self): |
|||
self.splash_ip = '127.0.0.1' |
|||
|
|||
''' |
|||
============================================================ |
|||
wait_for参数用来制定需要等待的元素，只有该元素渲染完成，程序才能染回，否则将等待200秒。wait_for 参数采购选择器的方式，如 |
|||
如制定元素id, 采用“#app"形式，如制定元素class, 采用 '.class-name'形式。 |
|||
============================================================ |
|||
''' |
|||
def post(self, url, wait_for, pages=1, page_element='', headers={'content-type':'application/json','Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8'}): |
|||
lua_scripts = """ |
|||
function wait_for_element(splash, css, maxwait) |
|||
-- Wait until a selector matches an element |
|||
-- in the page. Return an error if waited more |
|||
-- than maxwait seconds. |
|||
if maxwait == nil then |
|||
maxwait = 10 |
|||
end |
|||
return splash:wait_for_resume(string.format([[ |
|||
function main(splash) { |
|||
var selector = '%s'; |
|||
var maxwait = %s; |
|||
var end = Date.now() + maxwait*1000; |
|||
|
|||
function check() { |
|||
if(document.querySelector(selector)) { |
|||
splash.resume('Element found'); |
|||
} else if(Date.now() >= end) { |
|||
var err = 'Timeout waiting for element'; |
|||
splash.error(err + " " + selector); |
|||
} else { |
|||
setTimeout(check, 200); |
|||
} |
|||
} |
|||
check(); |
|||
} |
|||
]], css, maxwait)) |
|||
end |
|||
|
|||
function main(splash, args) |
|||
pages = """ + str(pages) + """ |
|||
page_element = '""" + page_element + """' |
|||
wait_for = '""" + wait_for + """' |
|||
splash:go('""" + url + """') |
|||
wait_for_element(splash, wait_for) |
|||
wait_for_element(splash, page_element) |
|||
|
|||
-- 将第一页的结果加入返回结果集中 |
|||
results = {splash.html()} |
|||
|
|||
if pages == 1 then |
|||
return results |
|||
else |
|||
-- 执行翻页动作 |
|||
-- 先页面上的翻页元件（element），然后发送点击事件（click()）翻页 |
|||
for i = 2, pages do |
|||
-- js 中是javascript脚本，用于获取翻页的元件，并发送click事件 |
|||
js = string.format("document.querySelector('%s').click();", page_element) |
|||
|
|||
-- 执行翻页脚本 |
|||
splash:runjs(js) |
|||
|
|||
-- 等待页面加载完成 |
|||
wait_for_element(splash, wait_for) |
|||
wait_for_element(splash, page_element) |
|||
|
|||
-- 这个地方看来必须加上延时，否则页面加载不完全，可能还没有完成页面更新 |
|||
assert(splash:wait(5)) |
|||
|
|||
-- 将页面加入返回结果集中 |
|||
table.insert(results, splash.html()) |
|||
end |
|||
return results |
|||
end |
|||
end |
|||
""" |
|||
|
|||
splash_url = 'http://' + self.splash_ip + ':8050/execute' |
|||
data = json.dumps({'lua_source':lua_scripts}) |
|||
r = HTMLSession().post(splash_url, headers=headers, data=data) |
|||
return r |
|||
|
|||
|
|||
class Crawler: |
|||
def __init__(self, connect): |
|||
self.connect = connect |
|||
|
|||
def generate_id(self): |
|||
# 用于生成一个32位的ID号 |
|||
current_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + str(random.randint(0, 1000000)) |
|||
md5_hash = hashlib.md5() |
|||
md5_hash.update(current_time.encode('utf-8')) |
|||
return md5_hash.hexdigest() |
|||
|
|||
def write_log_information(self, data_id, catalog_name, log_type='采购公告'): |
|||
# 添加了一条信息，需要同步更新其他相关信息, 包含对话框信息和日志信息两项 |
|||
with self.connect.cursor() as cursor: |
|||
affected_row = cursor.execute("select id from catalog where name = '%s'" % (log_type)) |
|||
if affected_row == 0: |
|||
return False |
|||
|
|||
result = cursor.fetchall() |
|||
catalog_id = result[0][0] |
|||
catalogdata_id = self.generate_id() |
|||
readlog_id = self.generate_id() |
|||
|
|||
affected_row = cursor.execute("SELECT staffid FROM userinfo where username = 'root'") |
|||
if affected_row == 0: |
|||
return False |
|||
|
|||
result = cursor.fetchall() |
|||
staff_id = result[0][0] |
|||
add_date = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") |
|||
|
|||
affected_row = cursor.execute( |
|||
'insert into catalogdata (id, dataid, catalogid, creatorid, menderid, adddate, modifydate, datastatus) values (%s, %s, %s, %s, %s, %s, %s, %s)', |
|||
(catalogdata_id, data_id, catalog_id, staff_id, staff_id, add_date, add_date, 0)) |
|||
|
|||
cursor.execute( |
|||
'insert into readlog (id, dataid, staffid, readnum, adddate, LastAccessDate, resid) values (%s, %s, %s, %s, %s, %s, %s)', |
|||
(readlog_id, data_id, staff_id, 1, datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), catalog_id)) |
|||
|
|||
return True |
|||
|
|||
|
|||
def write_information(self, title, url, region, publishTime, announcementType): |
|||
# 用于将一条信息写入数据库中 |
|||
with self.connect.cursor() as cursor: |
|||
cggg_id = self.generate_id() |
|||
|
|||
|
|||
try: |
|||
title = title.replace("'", "\\\'") |
|||
affected_rows = cursor.execute( |
|||
'insert into sc_cggg (id, bt, lj, ssqy, fbsj, gglb) values (%s, %s, %s, %s, %s, %s)', |
|||
(cggg_id, title, url, region, publishTime, announcementType)) |
|||
except pymysql.err.IntegrityError: |
|||
print('信息重复') |
|||
self.connect.rollback() |
|||
return False |
|||
else: |
|||
if self.write_log_information(cggg_id, announcementType): |
|||
self.connect.commit() |
|||
else: |
|||
print('添加采购信息失败') |
|||
self.connect.rollback() |
|||
return False |
|||
|
|||
return True |
|||
|
|||
def write_information_cgyx(self, cgyx): |
|||
# 用于将一条信息写入数据库中 |
|||
|
|||
with self.connect.cursor() as cursor: |
|||
cgyx_id = self.generate_id() |
|||
cgyx['cgxmmc'] = cgyx['cgxmmc'].replace("'", "\\\'") |
|||
strSql = 'insert into sc_cgyx (id, cgxmmc, lj, cgxqqk, ysje, yjcgsj, ly) values (\''+cgyx_id+'\',\''+cgyx['cgxmmc']+'\',\''+cgyx['lj']+'\',\''+cgyx['cgxqqk']+'\',\''+cgyx['ysje']+'\',\''+cgyx['yjcgsj']+'\',\''+cgyx['ly']+'\')' |
|||
try: |
|||
affected_rows = cursor.execute(strSql) |
|||
except pymysql.err.IntegrityError: |
|||
print('信息重复') |
|||
#self.connect.rollback() |
|||
return False |
|||
else: |
|||
if self.write_log_information(cgyx_id, '采购意向'): |
|||
self.connect.commit() |
|||
else: |
|||
print('添加采购信息失败') |
|||
self.connect.rollback() |
|||
return False |
|||
|
|||
return True |
|||
|
|||
def Check(self): |
|||
with self.connect.cursor() as cursor: |
|||
affected_row = cursor.execute("select id as total from sc_cggg where date(fbsj) > (NOW() - INTERVAL 1 DAY);") |
|||
if affected_row == 0: |
|||
gymailer.SendMail('jinqian_chen@126.com', 'jinqian.chen@srit.com.cn', '爬虫警告信息', '采购信息提取不正常，请检查！') |
|||
return False |
|||
else: |
|||
return True |
|||
|
|||
|
|||
def Crawl(self): |
|||
# 这个方法是实际完成爬取工作的总入口。 |
|||
|
|||
# 爬取浙江政采网的信息 |
|||
print('开始获取浙江政采网的信息\n') |
|||
|
|||
# 定义要传递进去的关于公告信息类型的数据结构 |
|||
infoType = [ |
|||
{"announcementCode": "110-175885", "announcementType":"采购意向"}, |
|||
{"announcementCode": "110-978863", "announcementType":"采购公告"}, |
|||
{"announcementCode": "110-943756", "announcementType":"更正公告"}, |
|||
{"announcementCode": "110-420383", "announcementType":"非政府采购公告"}, |
|||
{"announcementCode": "110-900461", "announcementType":"结果公告"} |
|||
] |
|||
for typeParam in infoType: |
|||
for page in range(1, 11): |
|||
try: |
|||
self.CrawlPage_zfcg_czt_zj(page, typeParam) |
|||
except Exception as e: |
|||
print('3--------------------------------', e) |
|||
|
|||
# 爬取宁波市阳光采购网的信息 |
|||
print('开始获取宁波市阳光采购网的信息\n') |
|||
infoType = [ |
|||
{"announcementCode": "21", "announcementType":"采购公告"}, |
|||
{"announcementCode": "23", "announcementType":"更正公告"}, |
|||
{"announcementCode": "22", "announcementType":"结果公告"} |
|||
] |
|||
for typeParam in infoType: |
|||
try: |
|||
self.CrawlPage_ygcg_nbcqjy_org(2, typeParam) |
|||
except Exception as e: |
|||
print('4--------------------------------', e) |
|||
|
|||
# 爬取宁波市中介超市网的信息 |
|||
print('开始获取宁波市中介超市网的信息\n') |
|||
infoType = [ |
|||
{"announcementCode": '1', "announcementType":"项目需求公告"}, |
|||
{"announcementCode": '2', "announcementType":"结果公告"} |
|||
] |
|||
|
|||
for typeParam in infoType: |
|||
for page in range(1, 6): |
|||
try: |
|||
self.CrawlPage_zjcs_nbxzfw(page, typeParam) |
|||
except Exception as e: |
|||
print('5------------------------------', e) |
|||
|
|||
# 爬取宁波市国资委市属企业采购信息 |
|||
print('开始获取宁波市国资委市属企业招投标网的信息\n') |
|||
for page in range(1, 5): |
|||
try: |
|||
self.CrawlPage_gzw_ningbo(page) |
|||
except Exception as e: |
|||
print('6------------------------------', e) |
|||
|
|||
# 爬取宁波中基国际招标网的信息 |
|||
print('开始获取宁波中基国际招标网的信息\n') |
|||
infoType = [ |
|||
{"announcementCode": "22", "announcementType":"采购公告"}, |
|||
{"announcementCode": "23", "announcementType":"结果公告"} |
|||
] |
|||
|
|||
for typeParam in infoType: |
|||
for page in range(1, 6): |
|||
try: |
|||
self.CrawlPage_cbbidding(page, typeParam) |
|||
except Exception as e: |
|||
print('7--------------------------------', e) |
|||
|
|||
# 爬取浙江国际招标网的信息 |
|||
print('开始获取浙江国际招标网的信息\n') |
|||
infoType = [ |
|||
{"announcementCode": "Zbgg", "announcementType":"采购公告"}, |
|||
{"announcementCode": "Gzgg", "announcementType":"更正公告"}, |
|||
{"announcementCode": "jggg", "announcementType":"结果公告"} |
|||
] |
|||
|
|||
for typeParam in infoType: |
|||
for page in range(1, 5): |
|||
try: |
|||
self.CrawlPage_zmeetb(page, typeParam) |
|||
except Exception as e: |
|||
print('8----------------------------', e) |
|||
|
|||
|
|||
# 爬取宁波市国际招标有限公司网站 |
|||
print('开始获取宁波国际招标网的信息\n') |
|||
|
|||
# 定义要传递进去的关于公告信息类型的数据结构 |
|||
infoType = [ |
|||
{"announcementCode": "1", "announcementType":"采购公告"}, |
|||
{"announcementCode": "1", "announcementType":"结果公告"}, |
|||
{"announcementCode": "2", "announcementType":"采购公告"}, |
|||
{"announcementCode": "2", "announcementType":"结果公告"} |
|||
] |
|||
for typeParam in infoType: |
|||
for page in range(1, 5): |
|||
try: |
|||
self.CrawlPage_nbbidding(page, typeParam) |
|||
except Exception as e: |
|||
print('9--------------------------------', e) |
|||
|
|||
# 爬取宁波名诚招标代理有限公司网站 |
|||
print('开始获取宁波名城招标的信息\n') |
|||
|
|||
# 定义要传递进去的关于公告信息类型的数据结构 |
|||
infoType = [ |
|||
{"announcementCode": "99", "announcementType":"采购公告"}, |
|||
{"announcementCode": "88", "announcementType":"结果公告"} |
|||
] |
|||
for typeParam in infoType: |
|||
for page in range(1, 2): |
|||
try: |
|||
self.CrawlPage_nbmcbidding(page, typeParam) |
|||
except Exception as e: |
|||
print('10--------------------------------', e) |
|||
|
|||
|
|||
# 宁波中基国际招标有限公司 https://www.cbbidding.com/ |
|||
def CrawlPage_cbbidding(self, page, typeParam): |
|||
# 这个方法是实际爬取指定页面的信息。 |
|||
session = HTMLSession() |
|||
session.DEFAULT_RETRIES = 5 |
|||
url = 'https://www.cbbidding.com/Index/cms.html?mid=' +typeParam['announcementCode'] + '&%2FIndex%2Fcms%2Fmid%2F' + typeParam['announcementCode'] + '_html=&page=' + str(page) |
|||
|
|||
headers = { |
|||
"Accept": "application/json, text/javascript, */*; q=0.01", |
|||
"Accept-Encoding": "gzip, deflate", |
|||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8", |
|||
"Connection": "keep-alive", |
|||
"DNT": '1', |
|||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36 HBPC/12.1.3.306" |
|||
} |
|||
|
|||
|
|||
# 这个网站返回的是一个网页，所以需要进行网页解析 |
|||
r = session.get(url = url, headers = headers) |
|||
|
|||
if r.status_code != 200: |
|||
if page == 1: |
|||
gymailer.SendMail('jinqian_chen@126.com', 'jinqian.chen@srit.com.cn', '爬虫警告信息:宁波中基国际招标网', r.text) |
|||
return False |
|||
|
|||
# 注意：xpath 函数返回的是list对象, 对象的元素是element |
|||
data = r.html.xpath('/html/body/div[3]/div[3]/div[2]/div[2]/div/ul/li') |
|||
for item in data: |
|||
title = item.xpath('//a')[0].text |
|||
url = 'https://www.cbbidding.com' + item.xpath('//a')[0].attrs.get('href') |
|||
region = '中基招标' |
|||
publishDate = item.xpath('//div')[0].text |
|||
|
|||
try: |
|||
publishDate = str(datetime.datetime.strptime(publishDate, '%Y-%m-%d')) |
|||
except Exception as e: |
|||
publishDate = publishDate.replace('.', '-') |
|||
publishDate = str(datetime.datetime.strptime(publishDate, '%Y-%m-%d')) |
|||
|
|||
print(url, title) |
|||
announcementType = typeParam['announcementType'] |
|||
#print(title, url, region, publishDate, announcementType) |
|||
self.write_information(title, url, region, publishDate, announcementType) |
|||
|
|||
|
|||
# 浙江国际招投标有限公司 https://www.zmeetb.com/ |
|||
def CrawlPage_zmeetb(self, page, typeParam): |
|||
# 这个方法是实际爬取指定页面的信息。 |
|||
session = HTMLSession() |
|||
url = 'https://www.zmeetb.com/' +typeParam['announcementCode'] + '/index/p/' + str(page) + '.html' |
|||
|
|||
headers = { |
|||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", |
|||
"Accept-Encoding": "gzip, deflate, br", |
|||
"Cache-Control": "max-age=0", |
|||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8", |
|||
"Connection": "close", |
|||
"DNT": '1', |
|||
"Host": "www.zmeetb.com", |
|||
"sec-ch-ua": '" Not A;Brand";v="99", "Chromium";v="99"', |
|||
"sec-ch-ua-mobile": "?0", |
|||
"sec-ch-ua-platform": "Windows", |
|||
"Sec-Fetch-Dest": "document", |
|||
"Sec-Fetch-Mode": "navigate", |
|||
"Sec-Fetch-Site": "none", |
|||
"Sec-Fetch-User": "?1", |
|||
"Upgrade-Insecure-Requests": "1", |
|||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36 HBPC/12.1.3.306" |
|||
} |
|||
|
|||
# 这个网站返回的是一个网页，所以需要进行网页解析 |
|||
# 这个网站如果使用render()函数，会遇到ssl证书问题，需要进一步研究chromium浏览器的证书问题 |
|||
#r = session.get(url = url, headers = headers, verify='/opt/PyGuoyan/www.zmeetb.com') |
|||
r = session.get(url = url, headers = headers, verify=False) |
|||
|
|||
if r.status_code != 200: |
|||
if page == 1: |
|||
gymailer.SendMail('jinqian_chen@126.com', 'jinqian.chen@srit.com.cn', '爬虫警告信息:浙江国际招标网', r.text) |
|||
return False |
|||
|
|||
# 注意：xpath 函数返回的是list对象, 对象的元素是element |
|||
data = r.html.xpath('/html/body/div[1]/div[3]/div[2]/div/div/div[3]/div/ul/li/a') |
|||
for item in data: |
|||
title = item.xpath('//p')[0].text |
|||
url = item.attrs.get('href') |
|||
region = '浙江国际招标' |
|||
publishDate = item.xpath('//p')[1].text |
|||
announcementType = typeParam['announcementType'] |
|||
|
|||
self.write_information(title, url, region, publishDate, announcementType) |
|||
|
|||
|
|||
# 宁波市名诚招标有限有限公司 http://www.nbmcbidding.com/ |
|||
def CrawlPage_nbmcbidding(self, page, typeParam): |
|||
# 这个方法是实际爬取指定页面的信息。 |
|||
session = HTMLSession() |
|||
if typeParam['announcementType'] == '采购公告': |
|||
url = "http://www.nbmcbidding.com/news/99/"+str(page)+"/" |
|||
else: |
|||
url = "http://www.nbmcbidding.com/news/88/"+str(page)+"/" |
|||
|
|||
|
|||
data = {} |
|||
headers = { |
|||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", |
|||
"Host": "www.nbmcbidding.com", |
|||
'Connection': 'keep-alive', |
|||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36 HBPC/12.1.3.306" |
|||
} |
|||
|
|||
r = session.get(url = url, headers = headers, json = data) |
|||
|
|||
if r.status_code != 200: |
|||
if page == 1: |
|||
gymailer.SendMail('jinqian_chen@126.com', 'jinqian.chen@srit.com.cn', '爬虫警告信息:宁波名诚招标代理有限公司', r.text) |
|||
return False |
|||
|
|||
# 注意：xpath 函数返回的是list对象, 对象的元素是element |
|||
data = r.html.xpath('/html/body/div[1]/div/div[3]/div[2]/ul/li') |
|||
for item in data: |
|||
title = item.xpath('//a/div[2]')[0].text |
|||
url = item.xpath('//a')[0].attrs.get('href') |
|||
region = '宁波名诚招标' |
|||
publishDate = item.xpath('//a/div[4]')[0].text |
|||
announcementType = typeParam['announcementType'] |
|||
|
|||
self.write_information(title, url, region, publishDate, announcementType) |
|||
|
|||
|
|||
# 宁波市国际招标有限公司 http://www.nbbidding.com/ |
|||
def CrawlPage_nbbidding(self, page, typeParam): |
|||
# 这个方法是实际爬取指定页面的信息。 |
|||
session = HTMLSession() |
|||
if typeParam['announcementType'] == '采购公告': |
|||
url = "http://www.nbbidding.com/Home/Notice/news_list?page="+str(page)+"&is_Open=1&keyword" |
|||
|
|||
else: |
|||
url = "http://www.nbbidding.com/Home/Publicity/news_list?page="+str(page)+"&is_Open=1&keyword" |
|||
|
|||
|
|||
data = {} |
|||
headers = { |
|||
"Accept": "application/json, text/javascript, */*; q=0.01", |
|||
"Host": "www.nbbidding.com", |
|||
'Connection': 'keep-alive', |
|||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36 HBPC/12.1.3.306" |
|||
} |
|||
|
|||
r = session.get(url = url, headers = headers, json = data) |
|||
|
|||
if r.status_code != 200: |
|||
if page == 1: |
|||
gymailer.SendMail('jinqian_chen@126.com', 'jinqian.chen@srit.com.cn', '爬虫警告信息:宁波市国际招标网', r.text) |
|||
return False |
|||
|
|||
data = json.loads(r.text)['data'] |
|||
total = data['page']['count'] |
|||
data = data['list'] |
|||
|
|||
for item in data: |
|||
id = item['id'] |
|||
if typeParam['announcementType'] == '采购公告': |
|||
url = 'http://www.nbbidding.com/Home/Notice/news_detail?id=%s' % (id) |
|||
else: |
|||
url = 'http://www.nbbidding.com/Home/Publicity/news_detail?id=%s' % (id) |
|||
title = item['title'] |
|||
region = '宁波国际招标' |
|||
publishDate = item['addtime'] |
|||
announcementType = item['stage'] |
|||
self.write_information(title, url, region, publishDate, announcementType) |
|||
|
|||
print(publishDate, title, url) |
|||
|
|||
# 宁波市国资委属企业招标信息网 |
|||
def CrawlPage_gzw_ningbo(self, page): |
|||
# 这个方法是实际爬取指定页面的信息。 |
|||
session = HTMLSession() |
|||
url = 'http://gzw.ningbo.gov.cn/col/col1229663137/index.html?uid=6085425&pageNum=%s' % str(page) |
|||
|
|||
headers = { |
|||
"Accept": "application/json, text/javascript, */*; q=0.01", |
|||
"Accept-Encoding": "gzip, deflate", |
|||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8", |
|||
"Connection": "keep-alive", |
|||
"DNT": '1', |
|||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36 HBPC/12.1.3.306" |
|||
} |
|||
|
|||
# 这个网站返回的是一个网页，所以需要进行网页解析 |
|||
r = session.get(url = url, headers = headers) |
|||
r.html.render() |
|||
|
|||
if r.status_code != 200: |
|||
if page == 1: |
|||
gymailer.SendMail('jinqian_chen@126.com', 'jinqian.chen@srit.com.cn', '爬虫警告信息:宁波市国资委市属企业招标信息网', r.text) |
|||
return False |
|||
|
|||
# 注意：xpath 函数返回的是list对象, 对象的元素是element |
|||
data = r.html.xpath('/html/body/div[2]/div[3]/div/div/div[2]/div/div/div/ul/li') |
|||
for item in data: |
|||
title = item.xpath('//a')[0].text |
|||
url = item.xpath('//a')[0].attrs.get('href') |
|||
region = '宁波市属国企' |
|||
publishDate = item.xpath('//p')[0].text |
|||
announcementType = '采购公告' |
|||
self.write_information(title, url, region, publishDate, announcementType) |
|||
|
|||
|
|||
|
|||
# 宁波市中介超市网 |
|||
def CrawlPage_zjcs_nbxzfw(self, page, typeParam): |
|||
# 这个方法是实际爬取指定页面的信息。 |
|||
# type 用于判别采购信息的类型 |
|||
session = HTMLSession() |
|||
urllist = ['http://zjcs.nbxzfw.gov.cn/newsweb/api/News/GetList?ClassId=0901&Type='+typeParam['announcementCode']+'&pageIndex='+str(page)+'&pageSize=15','http://zjcs.nbxzfw.gov.cn/newsweb/api/News/GetList?ClassId=0902&Type='+typeParam['announcementCode']+'&pageIndex='+str(page)+'&pageSize=15'] |
|||
|
|||
headers = { |
|||
"Accept": "application/json, text/javascript, */*; q=0.01", |
|||
"Accept-Encoding": "gzip, deflate", |
|||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8", |
|||
"Connection": "keep-alive", |
|||
"DNT": '1', |
|||
"Host": "ygcg.nbcqjy.org", |
|||
"Referer": "http://zjcs.nbxzfw.gov.cn/newsweb/page/news/infolist.html?Type="+str(type), |
|||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36 HBPC/12.1.3.306" |
|||
} |
|||
|
|||
for url in urllist: |
|||
r = session.get(url = url, headers = headers) |
|||
|
|||
if r.status_code != 200: |
|||
if page == 1: |
|||
gymailer.SendMail('jinqian_chen@126.com', 'jinqian.chen@srit.com.cn', '爬虫警告信息:宁波中介超市网', r.text) |
|||
return False |
|||
|
|||
data = json.loads(r.text)['data'] |
|||
|
|||
total = data['total'] |
|||
data = data['rows'] |
|||
|
|||
for item in data: |
|||
articleId = item['AutoId'] |
|||
BulletinTypeId = item['BulletinTypeId'] |
|||
url = 'http://zjcs.nbxzfw.gov.cn/YWGG/Info?id=%s&Type=%s' % (articleId, BulletinTypeId) |
|||
title = item['BulletinTitle'] |
|||
region = '宁波中介超市' |
|||
publishDate = item['PublishDate'] |
|||
announcementType = typeParam['announcementType'] |
|||
self.write_information(title, url, region, publishDate, announcementType) |
|||
|
|||
#print(publishDate, url) |
|||
|
|||
|
|||
# 宁波阳光采购网 |
|||
def CrawlPage_ygcg_nbcqjy_org(self, pages, typeParam): |
|||
url = 'https://ygcg.nbcqjy.org/list?type=2&class=%E5%85%AC%E5%91%8A%E5%85%AC%E7%A4%BA¬iceType=' + typeParam['announcementCode'] |
|||
|
|||
wait_for = '.ant-pagination-item-ellipsis' |
|||
page_element = '.anticon-right' |
|||
try: |
|||
r = Splash().post(url, wait_for, pages=pages, page_element=page_element) |
|||
except Exception as e: |
|||
print(e) |
|||
|
|||
results = json.loads(r.text) |
|||
|
|||
# 这个方法是实际爬取指定页面的信息。 |
|||
if r.status_code != 200: |
|||
if page == 1: |
|||
gymailer.SendMail('jinqian_chen@126.com', 'jinqian.chen@srit.com.cn', '爬虫警告信息:宁波阳光采购网, 错误代码：'+str(r.status_code), r.text) |
|||
return False |
|||
|
|||
for i in range(1, pages + 1): |
|||
data = HTML(html=results[str(i)]).xpath('/html/body/div/div/div[2]/div[2]/div/div/div[2]/div[2]/div[5]/div[1]/div/ul/li') |
|||
if len(data) == 0: |
|||
print('数据为空') |
|||
gymailer.SendMail('jinqian_chen@126.com', 'jinqian.chen@srit.com.cn', '爬虫警告信息:宁波阳光采购网, keyerror', e) |
|||
return False |
|||
|
|||
for item in data: |
|||
url = 'http://ygcg.nbcqjy.org' + item.xpath('//a')[0].attrs.get('href') |
|||
title = item.xpath('//a/span[3]')[0].text |
|||
region = '宁波阳光采购' |
|||
publishDate = item.xpath('//div[2]')[0].text |
|||
announcementType = typeParam['announcementType'] |
|||
print(title) |
|||
self.write_information(title, url, region, publishDate, announcementType) |
|||
|
|||
|
|||
# 浙江政府采购网 |
|||
def CrawlPage_zfcg_czt_zj(self, page, typeParam): |
|||
# 这个方法是实际爬取指定页面的信息。 |
|||
session = HTMLSession() |
|||
url = 'https://zfcg.czt.zj.gov.cn/portal/category' |
|||
if typeParam['announcementCode'] == '110-420383': |
|||
data = { |
|||
"pageNo": page, |
|||
"pageSize": 15, |
|||
"categoryCode": typeParam['announcementCode'], |
|||
"districtCode": ["339900"], |
|||
"isProvince": True, |
|||
"includeGovDistrict": "1", |
|||
"_t": 1699104836000 |
|||
} |
|||
else: |
|||
data = { |
|||
"pageNo": page, |
|||
"pageSize": 15, |
|||
"categoryCode": typeParam['announcementCode'], |
|||
"isGov": True, |
|||
"excludeDistrictPrefix": "90", |
|||
"_t": 1699104836000 |
|||
} |
|||
|
|||
headers = { |
|||
"accept": "application/json, text/plain, */*", |
|||
"accept-language": "zh-CN,zh;q=0.9,en;q=0.8", |
|||
"content-type": "application/json;charset=UTF-8", |
|||
"sec-ch-ua": "\" Not A;Brand\";v=\"99\", \"Chromium\";v=\"99\"", |
|||
"sec-ch-ua-mobile": "?0", |
|||
"sec-ch-ua-platform": "\"Windows\"", |
|||
"sec-fetch-dest": "empty", |
|||
"sec-fetch-mode": "cors", |
|||
"sec-fetch-site": "same-origin", |
|||
"x-requested-with": "XMLHttpRequest" |
|||
} |
|||
|
|||
try: |
|||
r = session.post(url = url, headers = headers, json = data) |
|||
except Exception as e: |
|||
print('10-------------------------', e) |
|||
|
|||
if r.status_code != 200: |
|||
if page == 1: |
|||
gymailer.SendMail('jinqian_chen@126.com', 'jinqian.chen@srit.com.cn', '爬虫警告信息:宁波政府采购网', r.text) |
|||
return False |
|||
|
|||
data = json.loads(r.text)['result']['data'] |
|||
total = data['total'] |
|||
data = data['data'] |
|||
|
|||
for item in data: |
|||
publishDate = datetime.datetime.fromtimestamp(item['publishDate']/1000) |
|||
pageUrl = 'https://zfcg.czt.zj.gov.cn/luban/detail?parentId=600007&articleId=' + item['articleId'] + '&utm=luban.luban-PC-37000.979-pc-websitegroup-zhejiang-secondPage-front.21.320086307d6811ee86314be74945ec2c' |
|||
detailUrl = 'https://zfcg.czt.zj.gov.cn/portal/detail?articleId=' + item['articleId'] |
|||
announcementType = typeParam['announcementType'] |
|||
if announcementType == '采购意向': |
|||
r = session.get(url = detailUrl, headers = headers) |
|||
|
|||
detailData = json.loads(r.text)['result']['data'] |
|||
if detailData == None: |
|||
break |
|||
|
|||
content = HTML(html='<xml>'+detailData['content']+'</xml>') |
|||
region = item['districtName'] |
|||
for detailItem in content.xpath('xml/div/div/div[1]/div/table/tbody/tr'): |
|||
title = detailItem.xpath('//td[2]')[0].text |
|||
cgxqqk = detailItem.xpath('//td[3]')[0].text |
|||
ysje = detailItem.xpath('//td[4]')[0].text |
|||
yjcgsj = detailItem.xpath('//td[5]')[0].text |
|||
ly = detailData["title"] |
|||
|
|||
self.write_information(title, pageUrl, region, publishDate, announcementType) |
|||
self.write_information_cgyx({'cgxmmc':title,'lj':pageUrl, 'cgxqqk':cgxqqk, 'ysje':ysje, 'yjcgsj':yjcgsj, 'ly':ly}) |
|||
else: |
|||
title = item['title'] |
|||
region = item['districtName'] |
|||
self.write_information(title, pageUrl, region, publishDate, announcementType) |
|||
|
|||
#print(publishDate, url) |
|||
|
|||
|
|||
return True |
|||
@ -0,0 +1,137 @@ |
|||
#!/usr/bin/python3 |
|||
|
|||
import pymysql |
|||
from properties import Properties |
|||
import sys, getopt |
|||
|
|||
class DbSearch: |
|||
# 本类用于提供各类数据库信息搜索服务 |
|||
def __init__(self, connect): |
|||
self.connect = connect |
|||
|
|||
def GetTableList(self, database): |
|||
# 查询某个库的数据表的列表 |
|||
cursorTable = self.connect.cursor() |
|||
cursorTable.execute("SELECT table_name FROM INFORMATION_SCHEMA.TABLES where table_schema = '" + database + "'"); |
|||
|
|||
return cursorTable.fetchall() |
|||
|
|||
def GetColumnList(self, tableName): |
|||
# 查询某张表的数据字段列表 |
|||
cursorColumn = self.connect.cursor() |
|||
cursorColumn.execute("SELECT column_name,data_type FROM INFORMATION_SCHEMA.COLUMNS where table_schema='" + database + "' AND table_name='" + |
|||
tableName + "'"); |
|||
return cursorColumn.fetchall() |
|||
|
|||
def SearchTableByColumnName(self, columnName, database): |
|||
# 查询包含包含searchText的库表 |
|||
tableList = self.GetTableList(database) |
|||
findList = list() |
|||
for table in tableList: |
|||
columnList = self.GetColumnList(table[0]) |
|||
for column in columnList: |
|||
if column[0].find(columnName) != -1: |
|||
findList.append(table[0]) |
|||
|
|||
return findList |
|||
|
|||
def SearchTableByText(self, searchText, database): |
|||
# 查找包含searchText字符串的表，并显示相应的表记录 |
|||
tableList = self.GetTableList(database) |
|||
if len(tableList) == 0: |
|||
return False |
|||
|
|||
found = 0 |
|||
findList = list() |
|||
for table in tableList: |
|||
strSql = "SELECT '" + table[0] + "' as table_name, t.* " |
|||
strSql = strSql + " FROM " + database + "." + table[0] + " as t where " + "(" |
|||
|
|||
columnList = self.GetColumnList(table[0]) |
|||
i = 0 |
|||
|
|||
count = len(columnList) |
|||
|
|||
for column in columnList: |
|||
# 如果字段数据类型为非文本型，跳过 |
|||
if not column[1] in ('varchar', 'char', 'text'): |
|||
continue |
|||
i += 1 |
|||
|
|||
if i > 1: |
|||
strSql += " or " |
|||
strSql += column[0] + " like '%" + searchText + "%' " |
|||
|
|||
strSql += ")" |
|||
|
|||
cursorColumn = self.connect.cursor() |
|||
try: |
|||
cursorColumn.execute(strSql) |
|||
except Exception as e: |
|||
print('2----------------------------', database, strSql) |
|||
print("-----错误信息：-----\n", e) |
|||
return False |
|||
|
|||
result = cursorColumn.fetchall() |
|||
if len(result) > 0: |
|||
findList.append(table[0]) |
|||
print("==========================================================================") |
|||
print(table[0], result, strSql) |
|||
return findList |
|||
|
|||
if __name__ == '__main__': |
|||
print( |
|||
""" |
|||
============================================================ |
|||
|这是数据库全文检索工具，包含两个参数 | |
|||
============================================================ |
|||
""") |
|||
|
|||
# 设置运行环境。如果当前是测试环境，则将is_test设置为true |
|||
is_test = False |
|||
|
|||
if is_test: |
|||
file_path = "/opt/eresource_test/webapp/WEB-INF/classes/prod/jdbc.properties" |
|||
database = 'guoyantest' |
|||
else: |
|||
file_path = "/opt/eresource/webapp/WEB-INF/classes/prod/jdbc.properties" |
|||
database = 'guoyan' |
|||
|
|||
# 打开jdbc.properties文件，获取数据库的配置信息 |
|||
props = Properties(file_path) |
|||
host = 'localhost' |
|||
user = props.get('jdbc.username') |
|||
password = props.get('jdbc.password') |
|||
|
|||
# 打开数据连接 |
|||
db = pymysql.connect(host = host, user = user, password = password, database = database) |
|||
|
|||
# 获取命令行参数 |
|||
keyword = '' |
|||
searchType ='' |
|||
|
|||
keyword = '' |
|||
searchType = '' |
|||
try: |
|||
opts, args = getopt.getopt(sys.argv[1:],"hT:k:",["keyword=","searchType="]) |
|||
except getopt.GetoptError: |
|||
print(sys.argv[0] + ' -k <keyword> -T <searchType>') |
|||
sys.exit(2) |
|||
|
|||
for opt, arg in opts: |
|||
if opt == '-h': |
|||
print('3--------------------', 'test.py -k <keyword> -T <searchType>') |
|||
sys.exit() |
|||
elif opt in ("-k", "--keyword"): |
|||
keyword = arg |
|||
elif opt in ("-T", "--searchType"): |
|||
searchType = arg |
|||
dbSearch = DbSearch(db) |
|||
if searchType == '0': |
|||
print('正在根据您输入的关键词查找表.....................') |
|||
print('found tables: ', dbSearch.SearchTableByText(keyword, database)) |
|||
elif searchType == '1': |
|||
print('正在根据您输入的列名查找表.....................') |
|||
print('found tables: ', dbSearch.SearchTableByColumnName(keyword, database)) |
|||
|
|||
|
|||
@ -0,0 +1,59 @@ |
|||
#!/usr/bin/python3 |
|||
"""这是爬虫的主程序主程序 |
|||
作者：陈进钱 |
|||
日期：2023/11/03 |
|||
""" |
|||
|
|||
import pymysql |
|||
import datetime |
|||
import time |
|||
from apscheduler.schedulers.blocking import BlockingScheduler |
|||
from properties import Properties |
|||
from crawler import Crawler |
|||
|
|||
print( |
|||
"""采购信息采集器 v1.0 |
|||
=================================================================================== |
|||
这个程序用于获取各大招投标网站的采购信息 |
|||
version: 1.0 |
|||
作者：陈进钱 |
|||
日期：2023-11-04 |
|||
===================================================================================""") |
|||
|
|||
# 设置运行环境。如果当前是测试环境，则将is_test设置为true |
|||
is_test = False |
|||
|
|||
if is_test: |
|||
file_path = "/opt/eresource_test/webapp/WEB-INF/classes/prod/jdbc.properties" |
|||
database = 'guoyantest' |
|||
else: |
|||
file_path = "/opt/eresource/webapp/WEB-INF/classes/prod/jdbc.properties" |
|||
database = 'guoyan' |
|||
|
|||
# 打开jdbc.properties文件，获取数据库的配置信息 |
|||
props = Properties(file_path) |
|||
host = 'localhost' |
|||
user = props.get('jdbc.username') |
|||
password = props.get('jdbc.password') |
|||
|
|||
# 打开数据连接 |
|||
connect = pymysql.connect(host = host, user = user, password = password, database = database) |
|||
|
|||
# 获取采购信息，并填写到数据库中 |
|||
crawler = Crawler(connect) |
|||
|
|||
# 启动自动爬取任务 |
|||
def crawl_check_func(): |
|||
crawler.Check() |
|||
|
|||
# 启动自动爬取任务 |
|||
def crawl_job_func(): |
|||
crawler.Crawl() |
|||
|
|||
sched = BlockingScheduler() |
|||
sched.add_job(crawl_job_func, 'interval', hours=3, jitter=120, max_instances=4) |
|||
sched.add_job(crawl_check_func, 'interval', days=1, jitter=120, max_instances=4) |
|||
sched.start() |
|||
|
|||
# 关闭数据库连接 |
|||
connect.close() |
|||
@ -0,0 +1,44 @@ |
|||
import smtplib |
|||
from email.mime.text import MIMEText |
|||
from email.header import Header |
|||
|
|||
def SendMail(sender, receiver, subject, message): |
|||
# 发送邮件服务器 |
|||
smtp_server = 'smtp.126.com' |
|||
|
|||
# 发送邮件服务器端口 |
|||
smtp_port = 465 |
|||
|
|||
# 邮件对象 |
|||
msg = MIMEText(message, 'plain', 'utf-8') |
|||
msg['From'] = Header(sender, 'utf-8') |
|||
msg['To'] = Header(receiver, 'utf-8') |
|||
msg['Subject'] = Header(subject, 'utf-8') |
|||
|
|||
# SMTP对象 |
|||
smtpObj = smtplib.SMTP_SSL(smtp_server, smtp_port) |
|||
|
|||
# 登录SMTP服务器 |
|||
smtpObj.login(sender, 'ERXYFJRLKPTTDXWH') |
|||
|
|||
# 发送邮件 |
|||
smtpObj.sendmail(from_addr=sender,to_addrs=[receiver],msg=msg.as_string()) |
|||
|
|||
# 关闭SMTP连接 |
|||
smtpObj.quit() |
|||
|
|||
if __name__ == '__main__': |
|||
|
|||
# 发件人邮箱 |
|||
sender = 'jinqian_chen@126.com' |
|||
|
|||
# 收件人邮箱 |
|||
receiver = 'jinqian.chen@srit.com.cn' |
|||
|
|||
# 邮件主题 |
|||
subject = 'Python3发送邮件示例, new' |
|||
|
|||
# 邮件正文 |
|||
message = '这是一封Python3发送的邮件' |
|||
SendMail(sender, receiver, subject, message) |
|||
|
|||
@ -0,0 +1,46 @@ |
|||
#Db2 |
|||
#hibernate.dialect=org.hibernate.dialect.DB2Dialect |
|||
#jdbc.driverClassName=com.ibm.db2.jcc.DB2Driver |
|||
#jdbc.url=jdbc:db2://localhost:50000/eaching |
|||
|
|||
#Oracle |
|||
#hibernate.dialect=org.hibernate.dialect.Oracle10gDialect |
|||
#jdbc.driverClassName=oracle.jdbc.driver.OracleDriver |
|||
#jdbc.url=jdbc:oracle:thin:@47.99.208.214:1521:orcl |
|||
#jdbc.url=jdbc:oracle:thin:@118.190.161.36:1521:orcl |
|||
|
|||
#SqlServer |
|||
#hibernate.dialect=org.hibernate.dialect.SQLServerDialect |
|||
#jdbc.driverClassName=net.sourceforge.jtds.jdbc.Driver |
|||
#jdbc.url=jdbc:jtds:sqlserver://localhost:1433/guanwaimatou;SelectMethod=Cursor |
|||
|
|||
|
|||
#MySql |
|||
hibernate.dialect=org.hibernate.dialect.MySQLDialect |
|||
jdbc.driverClassName=com.mysql.jdbc.Driver |
|||
jdbc.url=jdbc:mysql://116.62.210.190:3306/guoyantest?autoReconnect=true&useUnicode=true&characterEncoding=UTF8&mysqlEncoding=utf8&zeroDateTimeBehavior=convertToNull |
|||
|
|||
jdbc.username=root |
|||
jdbc.password=Guoyan83086775 |
|||
|
|||
jdbc.maxConn=20 |
|||
jdbc.minConn=5 |
|||
jdbc.activeTime=900000 |
|||
jdbc.alias=eaching |
|||
jdbc.keepingSleepTime=30000 |
|||
jdbc.maxConnectionLifetime=60000 |
|||
|
|||
jdbc.multiSource=false |
|||
|
|||
hibernate.cache.use_second_level_cache=true |
|||
hibernate.show_sql=false |
|||
hibernate.generate_statistics=false |
|||
hibernate.cache.provider_class=org.hibernate.cache.EhCacheProvider |
|||
#hibernate.cache.provider_class=net.oschina.j2cache.hibernate3.J2CacheProvider |
|||
hibernate.cache.use_minimal_puts=true |
|||
hibernate.cache.use_structured_entries=true |
|||
hibernate.cache.use_query_cache=true |
|||
hibernate.use_sql_comments=trues |
|||
hibernate.order_updates=true |
|||
hibernate.format_sql=false |
|||
hbm2ddl.auto=create |
|||
@ -0,0 +1,75 @@ |
|||
#!/usr/bin/python3 |
|||
"""这是爬虫的主程序主程序 |
|||
作者：陈进钱 |
|||
日期：2023/11/03 |
|||
""" |
|||
|
|||
import pymysql |
|||
import datetime |
|||
import time |
|||
from apscheduler.schedulers.blocking import BlockingScheduler |
|||
from properties import Properties |
|||
from crawler import Crawler |
|||
import sys |
|||
import os |
|||
|
|||
print( |
|||
"""采购信息采集器 v1.0 |
|||
=================================================================================== |
|||
这个程序用于获取各大招投标网站的采购信息 |
|||
version: 1.0 |
|||
作者：陈进钱 |
|||
日期：2023-11-04 |
|||
===================================================================================""") |
|||
|
|||
# 设置运行环境。如果当前是测试环境，则将is_test设置为true |
|||
is_test = True |
|||
if is_test: |
|||
root = "/opt/eresource_test/webapp/WEB-INF/classes/prod/" |
|||
else: |
|||
root = "/opt/eresource/webapp/WEB-INF/classes/prod/" |
|||
|
|||
if os.path.exists(root): |
|||
file_path = root + "jdbc.properties" |
|||
else: |
|||
file_path = "jdbc.properties" |
|||
|
|||
if sys.platform == 'win32': |
|||
host = '116.62.210.190' |
|||
user = 'root' |
|||
password = 'Guoyan83086775' |
|||
if is_test: |
|||
database = 'guoyantest' |
|||
else: |
|||
database = 'guoyan' |
|||
else: |
|||
if is_test: |
|||
database = 'guoyantest' |
|||
else: |
|||
database = 'guoyan' |
|||
|
|||
# 打开jdbc.properties文件，获取数据库的配置信息 |
|||
props = Properties(file_path) |
|||
host = '116.62.210.190' |
|||
user = props.get('jdbc.username') |
|||
password = props.get('jdbc.password') |
|||
|
|||
# 打开数据连接 |
|||
connect = pymysql.connect(host = host, user = user, password = password, database = database) |
|||
|
|||
# 获取采购信息，并填写到数据库中 |
|||
crawler = Crawler(connect) |
|||
crawler.Crawl() |
|||
#crawler.CrawlPage_ygcg_nbcqjy_org(1, {"announcementCode": "21", "announcementType":"采购公告"}) |
|||
#print(crawler.Check()) |
|||
|
|||
# 启动自动爬取任务 |
|||
#def crawl_job_func(): |
|||
# crawler.Crawl() |
|||
|
|||
#sched = BlockingScheduler() |
|||
#sched.add_job(crawl_job_func, 'interval', hours=1, jitter=120) |
|||
#sched.start() |
|||
|
|||
# 关闭数据库连接 |
|||
connect.close() |
|||
Binary file not shown.
@ -0,0 +1,72 @@ |
|||
#!/usr/bin/python |
|||
# -*- coding: UTF-8 -*- |
|||
|
|||
import re |
|||
import os |
|||
import tempfile |
|||
|
|||
|
|||
class Properties: |
|||
|
|||
def __init__(self, file_name): |
|||
# 如果配置文件不存在，取本地文件 |
|||
if not os.path.exists(file_name): |
|||
file_name = 'jdbc.properties' |
|||
|
|||
self.file_name = file_name |
|||
self.properties = {} |
|||
try: |
|||
fopen = open(self.file_name, 'r') |
|||
for line in fopen: |
|||
line = line.strip() |
|||
if line.find('=') > 0 and not line.startswith('#'): |
|||
strs = line.split('=') |
|||
self.properties[strs[0].strip()] = strs[1].strip() |
|||
except Exception as e: |
|||
raise e |
|||
else: |
|||
fopen.close() |
|||
|
|||
def has_key(self, key): |
|||
return key in self.properties |
|||
|
|||
def get(self, key, default_value=''): |
|||
if key in self.properties: |
|||
return self.properties[key] |
|||
return default_value |
|||
|
|||
def put(self, key, value): |
|||
self.properties[key] = value |
|||
replace_property(self.file_name, key + '=.*', key + '=' + value, True) |
|||
|
|||
|
|||
def replace_property(file_name, from_regex, to_str, append_on_not_exists=True): |
|||
tmpfile = tempfile.TemporaryFile() |
|||
|
|||
if os.path.exists(file_name): |
|||
r_open = open(file_name, 'r') |
|||
pattern = re.compile(r'' + from_regex) |
|||
found = None |
|||
for line in r_open: |
|||
if pattern.search(line) and not line.strip().startswith('#'): |
|||
found = True |
|||
line = re.sub(from_regex, to_str, line) |
|||
tmpfile.write(line.encode()) |
|||
if not found and append_on_not_exists: |
|||
tmpfile.write(('\n' + to_str).encode()) |
|||
r_open.close() |
|||
tmpfile.seek(0) |
|||
|
|||
content = tmpfile.read() |
|||
|
|||
if os.path.exists(file_name): |
|||
os.remove(file_name) |
|||
|
|||
w_open = open(file_name, 'wb') |
|||
w_open.write(content) |
|||
w_open.close() |
|||
|
|||
tmpfile.close() |
|||
else: |
|||
print ("file %s not found" % file_name) |
|||
|
|||
@ -0,0 +1,14 @@ |
|||
#!/usr/bin/python3 |
|||
from splash.gysplash import SBase |
|||
import json |
|||
|
|||
class SYgcg(SBase): |
|||
def open(self): |
|||
return super().open('ygcg', pages=2, annoucement_type='政府采购') |
|||
|
|||
if __name__ == '__main__': |
|||
test = SYgcg() |
|||
r = test.open() |
|||
|
|||
results = json.loads(r.text) |
|||
print(results) |
|||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -0,0 +1,7 @@ |
|||
#This is splash config file. |
|||
[splash service settings] |
|||
server = localhost |
|||
port = 8050 |
|||
|
|||
|
|||
|
|||
@ -0,0 +1,20 @@ |
|||
{ |
|||
"description": "This is splash config file.", |
|||
"server": "127.0.0.1", |
|||
"port": "8050", |
|||
"class":{ |
|||
"SYgcg":{ |
|||
"url":"https://ygcg.nbcqjy.org/list?type=2&class=%E5%85%AC%E5%91%8A%E5%85%AC%E7%A4%BA¬iceType={{ $noticeType }}", |
|||
"_comment":"http://www.baidu.com", |
|||
"param":{ |
|||
"noticeType":"21" |
|||
}, |
|||
"wait_for":".ant-list-items", |
|||
"page_element":".anticon-right", |
|||
"headers":{ |
|||
"content-type":"application/json", |
|||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0" |
|||
} |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1 @@ |
|||
This is lua script file |
|||
@ -0,0 +1,122 @@ |
|||
#!/usr/bin/python3 |
|||
'''=================================================================== |
|||
这个模块用于对 splash 服务进行封装，方便在 python 中使用。 |
|||
版本：1.0 |
|||
作者：陈进钱 |
|||
日期：2023-12-18 |
|||
===================================================================''' |
|||
import os |
|||
import re |
|||
import json |
|||
import codecs |
|||
import configparser |
|||
from requests_html import HTMLSession |
|||
from requests_html import HTML |
|||
|
|||
config = configparser.ConfigParser() |
|||
# splash 基类 |
|||
class SBase: |
|||
def __init__(self): |
|||
self.__lua_script = '' |
|||
self.config = {} |
|||
|
|||
# 创建 ConfigParser 对象 |
|||
self.root = os.path.dirname(os.path.abspath(__file__)) |
|||
|
|||
# 自动创建配置文件 |
|||
dir = self.root + '/config' |
|||
if not os.path.exists(dir): |
|||
os.makedirs(dir) |
|||
|
|||
file_path = self.root + '/config/splash.json' |
|||
|
|||
if os.path.exists(file_path): |
|||
file = codecs.open(file_path, 'r', 'utf-8') |
|||
content = file.read() |
|||
self.config = json.loads(content) |
|||
file.close() |
|||
else: |
|||
self.config['description'] = 'This is splash config file.' |
|||
self.config['server'] = 'localhost' |
|||
self.config['port'] = '8050' |
|||
|
|||
content = json.dumps(self.config) |
|||
with codecs.open(file_path, 'w', 'utf-8') as file: |
|||
file.write(content) |
|||
|
|||
# 自动创建空的脚本文件 |
|||
dir = self.root + '/scripts' |
|||
if not os.path.exists(dir): |
|||
os.makedirs(dir) |
|||
|
|||
# 这个代码要更新为一个通用代码 |
|||
file_path = dir + '/main.lua' |
|||
|
|||
if os.path.exists(file_path): |
|||
file = codecs.open(file_path, 'r', 'utf-8') |
|||
self.__lua_script = file.read() |
|||
file.close() |
|||
else: |
|||
with codecs.open(file_path, 'w', 'utf-8') as file: |
|||
self.__lua_script = 'This is lua script file' |
|||
file.write(self.__lua_script) |
|||
|
|||
def script(self): |
|||
return self.__lua_script; |
|||
|
|||
def class_name(self): |
|||
return type(self).__name__; |
|||
|
|||
def replace(self, source, param, value): |
|||
return re.sub('{{[\s]*\$' + param + '[\s]*}}', value, source) |
|||
|
|||
# 向lua脚本传递参数变量 |
|||
def set_params_for_lua(self, scripts, params): |
|||
for param in params: |
|||
scripts = self.replace(scripts, param, params[param]) |
|||
return scripts |
|||
|
|||
''' |
|||
-------------------------------------------------------------------------------------- |
|||
本函数用于打开指定的网址。具体的网址、参数、等待就绪的网页元件、等待就绪的翻页元件、 |
|||
headers 等参数默认为空。这些参数的任意一个为空时，则从配置文件中的相关类名项下提取。 |
|||
本函数会调用 lua 主脚本文件，执行页面解析的lua脚本文件。该文件名称通过参数 parser 传递。 |
|||
-------------------------------------------------------------------------------------- |
|||
''' |
|||
def open(self): |
|||
pass |
|||
def open(self, scripts_js, pages=1, url='', params=None, |
|||
wait_for='', page_element='', headers='', annoucement_type=''): |
|||
if url == '': |
|||
url = self.config['class'][self.class_name()]['url'] |
|||
if params == None: |
|||
params = self.config['class'][self.class_name()]['param'] |
|||
if len(params) > 0: |
|||
for param in params: |
|||
url = self.replace(url, param, params[param]) |
|||
|
|||
if wait_for == '': |
|||
wait_for = self.config['class'][self.class_name()]['wait_for'] |
|||
|
|||
if page_element =='': |
|||
page_element = self.config['class'][self.class_name()]['page_element'] |
|||
|
|||
if headers == '': |
|||
headers = self.config['class'][self.class_name()]['headers'] |
|||
scripts = self.script() |
|||
scripts = self.set_params_for_lua(scripts, { |
|||
'pages':str(pages), |
|||
'url':url, |
|||
'wait_for':wait_for, |
|||
'page_element':page_element, |
|||
# 这个解析器要从通过参数传递 |
|||
'scripts_js': scripts_js, |
|||
'announcement_type':annoucement_type |
|||
}) |
|||
|
|||
# print(scripts) |
|||
data = json.dumps({'lua_source':scripts}) |
|||
splash_url = 'http://' + self.config['server'] + ':' + self.config['port'] + '/execute' |
|||
r = HTMLSession().post(splash_url, headers=headers, data=data) |
|||
|
|||
return r |
|||
@ -0,0 +1,75 @@ |
|||
-- 本文件是页面抓取的主入口 |
|||
-- 这里必须采用加载模块的方法，否则好像不能动态加载js文件 |
|||
parser = require('parser') |
|||
|
|||
function main(splash, args) |
|||
pages = {{$pages}} |
|||
scripts_js = '{{$scripts_js}}' |
|||
page_element = '{{$page_element}}' |
|||
wait_for = '{{$wait_for}}' |
|||
announcement_type = '{{$announcement_type}}' |
|||
splash:go('{{$url}}') |
|||
wait_for_element(splash, wait_for) |
|||
wait_for_element(splash, page_element) |
|||
|
|||
-- 设置javascript脚本参数 |
|||
results = {} |
|||
params_js = {} |
|||
params_js['announcement_type'] = announcement_type |
|||
|
|||
-- 将第一页的结果加入返回结果集中 |
|||
result = parser.select(splash, scripts_js, params_js) |
|||
table.insert(results, result) |
|||
|
|||
if pages == 1 then |
|||
return results |
|||
else |
|||
-- 执行翻页动作 |
|||
-- 先页面上的翻页元件（element），然后发送点击事件（click()）翻页 |
|||
for i = 2, pages do |
|||
-- 执行翻页脚本 |
|||
-- js 中是javascript脚本，用于获取翻页的元件，并发送click事件 |
|||
js = string.format("document.querySelector('%s').click();", page_element) |
|||
splash:runjs(js) |
|||
|
|||
-- 等待页面加载完成 |
|||
wait_for_element(splash, wait_for) |
|||
wait_for_element(splash, page_element) |
|||
|
|||
-- 这个地方看来必须加上延时，否则页面加载不完全，可能还没有完成页面更新 |
|||
assert(splash:wait(5)) |
|||
result = parser.select(splash, scripts_js, params_js) |
|||
table.insert(results, result) |
|||
end |
|||
return results |
|||
end |
|||
end |
|||
|
|||
function wait_for_element(splash, css, maxwait) |
|||
-- Wait until a selector matches an element |
|||
-- in the page. Return an error if waited more |
|||
-- than maxwait seconds. |
|||
if maxwait == nil then |
|||
maxwait = 10 |
|||
end |
|||
return splash:wait_for_resume(string.format([[ |
|||
function main(splash) { |
|||
var selector = '%s'; |
|||
var maxwait = %s; |
|||
var end = Date.now() + maxwait*1000; |
|||
|
|||
function check() { |
|||
if(document.querySelector(selector)) { |
|||
splash.resume('Element found'); |
|||
} else if(Date.now() >= end) { |
|||
var err = 'Timeout waiting for element'; |
|||
splash.error(err + " " + selector); |
|||
} else { |
|||
setTimeout(check, 200); |
|||
} |
|||
} |
|||
check(); |
|||
} |
|||
]], css, maxwait)) |
|||
end |
|||
|
|||
File diff suppressed because one or more lines are too long
@ -0,0 +1,28 @@ |
|||
-- 文件名为 module.lua |
|||
-- 定义一个名为 module 的模块 |
|||
parser = {} |
|||
|
|||
function set_params(scripts, params_js) |
|||
for param, value in pairs(params_js) do |
|||
scripts = scripts.gsub(scripts, "{{(%s*)$" .. param .. "(%s*)}}", value) |
|||
end |
|||
--scripts = scripts.gsub('123456 aaaa 123456', "[\s\\\]*aaaa\\\[\\\\s\\\]*", 'bbbb') |
|||
return scripts |
|||
end |
|||
|
|||
-- 定义一个函数 |
|||
function parser.select(splash, scripts_js, params_js) |
|||
local file = io.open("/etc/splash/lua_modules/jquery-3.7.1.min.js", "r") |
|||
splash:runjs(file:read('*a')) |
|||
file:close() |
|||
|
|||
file = assert(io.open("/etc/splash/lua_modules/"..scripts_js..".js", "r")) |
|||
scripts = file:read('*a') |
|||
scripts = set_params(scripts, params_js) |
|||
local js = splash:jsfunc(scripts) |
|||
file:close() |
|||
|
|||
return js() |
|||
end |
|||
|
|||
return parser |
|||
@ -0,0 +1,32 @@ |
|||
function () { |
|||
title = ''; |
|||
url = ''; |
|||
updateTime = ''; |
|||
region = ''; |
|||
announcementType = ''; |
|||
results = {}; |
|||
lists = new Array(); |
|||
|
|||
// 取列表的头
|
|||
ul = $('#app > div > div.z_list_vue > div.ant-spin-nested-loading > div > div > div.z_content > div.z_detail_content > div:nth-child(5) > div.ant-spin-nested-loading > div > ul'); |
|||
// 获取列表的第一个元素，获取成功的话，元素封装对象的length = 1
|
|||
li = ul.children('li').first() |
|||
item = {} |
|||
while (li.length == 1) |
|||
{ |
|||
a = li.find('div.ant-list-item-meta > div > h4 > span > a'); |
|||
item.title = $(a.children()['2']).attr('title'); |
|||
item.url = a.attr('href'); |
|||
item.updateTime = $(li.children()[1]).text(); |
|||
item.region = '宁波阳光采购'; |
|||
item.announcementType = '{{$announcement_type}}' |
|||
|
|||
lists.push(item) |
|||
// 取下一个列表元素
|
|||
li = li.next() |
|||
} |
|||
|
|||
results.count = lists.length |
|||
results.lists = lists |
|||
return results |
|||
} |
|||
File diff suppressed because it is too large
File diff suppressed because it is too large
Loading…
Reference in new issue