主体代码和老师提供差不多，就是将分页封装了方法，通过递归方式调用
import requests
from lxml import etree
import json
# 一、抓取首页-资讯菜单下面13个频道名称 url
# 二、分析每个频道页的全部资讯抓取策略
# 在20题的基础上抓取每个频道每个资讯的标题 资讯的内容详情
def haha(page,item):
post_url='https://www.huxiu.com/channel/ajaxGetMore'
channel_name=item['channel_name']
catId=item['catId']
post_data={
'huxiu_hash_code':'18f3ca29452154dfe46055ecb6304b4e',
'page':page,
'catId':catId
}
html=requests.post(post_url,data=post_data,headers=headers).text
dict_data=json.loads(html)
parse_data=dict_data['data']
total_page=parse_data['total_page']
data=parse_data['data']
# print(data)
selector=etree.HTML(data)
article_urls=selector.xpath('//a/@href')
for article_url in article_urls:
if article_url.startswith('/article'):
article_url=root_url+article_url
print(channel_name, catId, total_page, page, article_url)
if(total_page>int(page)):
page = int(page)+1
haha(page,item)
else:
pass
root_url='https://www.huxiu.com'
headers={
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36',
}
html=requests.get(root_url,headers=headers).text
selector=etree.HTML(html)
zixun_infos=selector.xpath('//ul[@class="header-column header-column1 header-column-zx menu-box"]/li/a')
items=[]
for info in zixun_infos:
item={}
channel_name=info.xpath('text()')[0]
catId=info.xpath('@href')[0].replace('/channel/','').replace('.html','')
# print(channel_name,catId)
item['channel_name']=channel_name
item['catId']=catId
items.append(item)
for item in items:
haha('1',item)
