简单描述一下爬取的基本思路：
- 在google上搜索海贼王，选定风之动漫网为目标进行爬取数据,如:http://manhua.fzdm.com/2/846/index_1.html
- 观察每个页面url规律，846是代表话数，index_page.html代表是多少页
- 检查页面的图片便签，找出唯一能指定该图片的CSS表达式
- 使用requests来get到页面的报文，使用BeautifulSoup来解析报文
- 原计划使用MongoDB存储图片地址，处于暂时操作mongodb还不够熟练，直接使用了列表操作
- 使用urllib来进行下载图片到本地
from bs4 import BeautifulSoup
import requests
import time
import pymongo
import urllib.request
import os
path = '/Users/meixuhong/OnePiece/'
# ================================== 设计数据库 ====================================
client = pymongo.MongoClient('localhost',27017)
onepiece = client['onepiece']
onepiece_pic = onepiece['onepiece_pic']
# ================================== 抓取多页数据 ==================================
def parseMultiplePages(chapter,page_num):
img_urls = []
for page_num in range(1,page_num+1):
time.sleep(4)
wb_data = requests.get('http://manhua.fzdm.com/2/{}/index_{}.html'.format(chapter,page_num))
soup = BeautifulSoup(wb_data.text,'lxml')
imgs = soup.select('div#mh > li > a > img')
for img in imgs:
data = {
'img': img.get('src')
}
print(data)
# onepiece_pic.insert_one(data)
img_urls.append(data['img'])
print('img_urls is a list as:',img_urls)
return img_urls
# 837话的前16页
# parseMultiplePages(837,16)
# ================================== 下载漫画并命名 ==================================
def dl_images(chapter,img_urls):
#==判断并创建目录==
subPath = path + str(chapter) + '/'
isExists = os.path.exists(subPath)
if not isExists:
print('create the path: {}...'.format(subPath))
os.mkdir(subPath)
else:
print('the path already exsiting ...')
# ==判断并创建目录==
for i in range(1,len(img_urls)+1):
# 使用urllib.request.urlretrieve(url, fine_path_name)下载文件
urllib.request.urlretrieve(img_urls[i-1],subPath+str(i)+'_'+img_urls[i-1].split('/')[-1])
print('\n{} downloaded and has been named as {}.\n'.format(img_urls[i-1],subPath+str(i)+'_'+img_urls[i-1].split('/')[-1]))
# ================================== 下载多话漫画 ==================================
def dl_chapters(chapter_from_,chapter_to_):
for i in range(chapter_from_ , chapter_to_ + 1):
dl_images(i,parseMultiplePages(i,18))
dl_chapters(800,848)
程序完全只考虑了功能实现，没有考虑多做考虑，以后海贼王更新的时候不用到处找资源慢慢等待了，满足我个人需求。
OnePiece
