130行代码实现海贼王漫画下载

ゝ一世哀愁。 2022-02-03 11:59 206阅读 0赞

创客学院小编给大家带来一点福利，但不是妹子图！不是妹子图！不是妹子图！  
敲了130多行代码，利用协程实现漫画下载，亲测没问题，目前海贼王更新到930话，全部下载下来1小时左右，供大家参考，一起共勉。  
代码烂了 大神别笑话我。

from gevent import monkey;monkey.patch\_all()  
from gevent.pool import Pool  
from bs4 import BeautifulSoup  
from fake\_useragent import UserAgent  
from requests.packages.urllib3.exceptions import InsecureRequestWarning

import gevent  
import requests  
import time  
import os  
import shutil

def getSource(urls, headers, types):  
try:  
\# 实例化UserAgent类  
user\_agent = UserAgent()  
\# 为头文件随机分配User-Agent  
headers\['User-Agent'\] = user\_agent.random  
\# 禁用安全请求警告  
requests.packages.urllib3.disable\_warnings(InsecureRequestWarning)  
\# 实例化Session  
request\_session = requests.Session()  
\# 设置重连次数  
request\_session.mount('http://', requests.adapters.HTTPAdapter(max\_retries=5))  
request\_session.mount('https://', requests.adapters.HTTPAdapter(max\_retries=5))  
\# 执行请求  
get\_response = request\_session.get(urls, headers=headers, verify=False, timeout=(10, 10))  
\# 关闭请求  
request\_session.close()  
\# 设置编码  
get\_response.encoding = 'UTF-8'  
\# 判断获取源码还是图片  
if types == 'text':  
get\_response = get\_response.text  
if types == 'content':  
get\_response = get\_response.content  
except Exception as e:  
print('getSource()函数异常：' + str(e))  
else:  
return get\_response

def sourceAnalysis(src, dic, typ):  
\# 定义章节链接、标题、内容列表  
chapter\_link = \[\]  
chapter\_name = \[\]  
chapter\_cont = \[\]  
\# 实例化BeautifulSoup  
soup = BeautifulSoup(src, 'html.parser')  
\# 解析章节链接和标题  
if typ == 'chapter':  
analysis\_lists = soup.find\_all(dic\['label'\], class\_=dic\['class'\])  
\# 提取章节链接和标题  
for i in range(len(analysis\_lists)):  
chapter\_link.append(DOMAIN + analysis\_lists\[i\].get('data-hreflink'))  
chapter\_name.append(analysis\_lists\[i\].get\_text().strip())  
chapter\_dic = \{'chapter\_link': chapter\_link, 'chapter\_name': chapter\_name\}  
return chapter\_dic  
\# 解析章节内图片链接  
if typ == 'content':  
analysis\_lists = soup.find\_all(dic\['label'\], class\_=dic\['class'\])  
\# 提取章节内图片链接  
for i in range(len(analysis\_lists)):  
chapter\_cont.append(analysis\_lists\[i\].get('data-src'))  
return chapter\_cont

if \_\_name\_\_ == '\_\_main\_\_':  
\# 系统启动时间  
start\_time = time.time()

\# 定义常量  
DOMAIN = 'https://www.mkzhan.com/'  
REQUEST\_URL = 'https://www.mkzhan.com/209871/'  
HEADERS = \{'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,\*/\*;q=0.8',  
'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',  
'Connection': 'keep-alive',  
'User-Agent': ''\}  
LINK\_PROPERTY = \{'label': 'a', 'class': 'j-chapter-link'\}  
IMAG\_PROPERTY = \{'label': 'img', 'class': 'lazy-read'\}  
POOL = Pool(100)  
ROOT\_PATH = "D:/OnePiece/"

\# 创建存储漫画文件夹，如果已有文件夹，则删除再新建  
if os.path.exists(ROOT\_PATH):  
shutil.rmtree(ROOT\_PATH)  
os.mkdir(ROOT\_PATH)

\# 获取目录页源码  
function\_run\_time = time.time()  
print(time.strftime('\[%Y-%m-%d %H:%M:%S\]', time.localtime()) + " 获取目录页源码开始...")  
catalog\_source = getSource(REQUEST\_URL, HEADERS, 'text')  
print(time.strftime('\[%Y-%m-%d %H:%M:%S\]', time.localtime()) + " 获取目录页源码完成...\[ %.1fs \]" % (time.time() - function\_run\_time))

\# 解析章节信息  
function\_run\_time = time.time()  
print(time.strftime('\[%Y-%m-%d %H:%M:%S\]', time.localtime()) + " 解析章节信息开始...")  
chapter\_info = sourceAnalysis(catalog\_source, LINK\_PROPERTY, 'chapter')  
print(time.strftime('\[%Y-%m-%d %H:%M:%S\]', time.localtime()) + " 解析章节信息完成...\[ %.1fs \]" % (time.time() - function\_run\_time))

\# 获取每章节源码  
function\_run\_time = time.time()  
print(time.strftime('\[%Y-%m-%d %H:%M:%S\]', time.localtime()) + " 获取每章节源码开始...")  
get\_source\_worker = \[POOL.spawn(getSource, url, HEADERS, 'text') for url in chapter\_info\['chapter\_link'\]\]  
gevent.joinall(get\_source\_worker)  
chapter\_source = \[source.value for source in get\_source\_worker\]  
print(time.strftime('\[%Y-%m-%d %H:%M:%S\]', time.localtime()) + " 获取每章节源码完成...\[ %.1fs \]" % (time.time() - function\_run\_time))

\# 解析章节内图片链接  
function\_run\_time = time.time()  
print(time.strftime('\[%Y-%m-%d %H:%M:%S\]', time.localtime()) + " 解析章节内图片链接开始...")  
get\_imglink\_worker = \[POOL.spawn(sourceAnalysis, src, IMAG\_PROPERTY, 'content') for src in chapter\_source\]  
gevent.joinall(get\_imglink\_worker)  
image\_list = \[link.value for link in get\_imglink\_worker\]  
print(time.strftime('\[%Y-%m-%d %H:%M:%S\]', time.localtime()) + " 解析章节内图片链接完成...\[ %.1fs \]" % (time.time() - function\_run\_time))

\# 下载漫画  
for i in range(len(chapter\_info\['chapter\_name'\])):  
function\_run\_time = time.time()  
print(time.strftime('\[%Y-%m-%d %H:%M:%S\]', time.localtime()) + " 下载 " + chapter\_info\['chapter\_name'\]\[i\] + " 开始...")  
get\_images\_worker = \[POOL.spawn(getSource, url, HEADERS, 'content') for url in image\_list\[i\]\]  
gevent.joinall(get\_images\_worker)  
\# 创建章节文件夹  
save\_path = ROOT\_PATH + chapter\_info\['chapter\_name'\]\[i\] + '/'  
os.mkdir(save\_path)  
for j in range(len(get\_images\_worker)):  
with open(save\_path + str(j) + '.jpg', 'wb') as image:  
image.write(get\_images\_worker\[j\].value)  
print(time.strftime('\[%Y-%m-%d %H:%M:%S\]', time.localtime()) + " 下载 " + chapter\_info\['chapter\_name'\]\[i\] + " 完成...\[ %.1fs \]" % (time.time() - function\_run\_time))

print(time.strftime('\[%Y-%m-%d %H:%M:%S\]', time.localtime()) + ' System executing done...\[ %.1fs \]' % (time.time() - start\_time))

Python群：399932895