小案例
# pip3 install beautifulsoup4
from bs4 import BeautifulSoup
import requests,re
#http://all.hengyan.com/1/0_0_0_0_0_0_0_0_0_1.aspx
#http://all.hengyan.com/1/0_0_0_0_0_0_0_0_0_2.aspx
#http://all.hengyan.com/1/0_0_0_0_0_0_0_0_0_3.aspx
#http://all.hengyan.com/1/0_0_0_0_0_0_0_0_0_100.aspx
class HengYanSpider(object):
def __init__(self):
self.first_url = 'http://all.hengyan.com/1/0_0_0_0_0_0_0_0_0_1.aspx'
self.default_headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36'
}
def get_noval_url(self,url=None):
url = self.first_url if not url else url
"""获取小说详情的url地址"""
html = self.send_request(url)
if html:
###使用bs4解析数据
bs_soup = BeautifulSoup(html,'lxml')
lis = bs_soup.find_all(name='li',attrs={'class':'bookname'})
print(lis)
for li in lis:
a_list = li.select('a')
if len(a_list) > 0:
url = a_list[0].attrs['href']
print(url)
self.get_noval_detail(url)
else:
print('数据获取失败')
def get_noval_detail(self,noval_url):
"""获取书籍详情的页面内容,解析数据"""
html = self.send_request(noval_url)
if html:
print('得到了详情页面'）
####使用bs4解析
noval_dict = {}
bs_soup = BeautifulSoup(html,'lxml')
#//div[@class="dh"]/p/label/text()
# #书号
book_id = bs_soup.select('div.dh p label')[0].get_text()
noval_dict['book_id'] = re.search('\d+',book_id).group()
# #火车票
# //div[@class="piao"]/p[2]/span[@class="huocolor"]/text()
noval_dict['hot_track'] = bs_soup.select('.piao p')[1].select('.huocolor')[0].get_text()
print(noval_dict)
# print(noval_dict)
# self.save_data(noval_dict)
def save_data(self,noval_dict):
"""保存数据"""
pass
def extract_first(self,data,default=''):
if len(data) > 0:
return data[0]
return default
def send_request(self,url,header=None,data=None,method="GET"):
"""发送请求"""
header = self.default_headers if not header else header
if method == 'GET':
#发送get请求
response = requests.get(url=url,params=data,headers=header)
else:
#发送post请求
response = requests.post(url=url, data=data, headers=header)
if response.status_code == 200:
#请求成功，返回页面源码
return response.text
if __name__ == '__main__':
spider = HengYanSpider()
spider.get_noval_url()
