这里是最近几年写过的爬虫,做一个分享
1. NASA Mars图片爬虫
NASA的图片是可以通过接口获取到,这就很简单,我们只需要访问接口去获取就行了
程序运行截图
爬取的图片
import requests
import json
headers = {
"user-agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36"
}
params = {
'size':100,
'from':100 ,
'sort':'promo-date-time:desc',
'q':'((ubernode-type:image) AND (topics:3152))',
'_source_include':'promo-date-time,master-image,nid,title,topics,missions,collections,other-tags,ubernode-type,primary-tag,secondary-tag,cardfeed-title,type,collection-asset-link,link-or-attachment,pr-leader-sentence,image-feature-caption,attachments,uri'
}
# https://www.nasa.gov/sites/default/files/thumbnails/image/pia23454.jpg
public_path = "https://www.nasa.gov/sites/default/files/"
rsp=requests.get('https://www.nasa.gov/api/2/ubernode/_search',headers=headers,params=params)
# print(rsp.text)
b = json.loads (rsp.text)
hits = b['hits']['hits']
for h in hits:
_source = h['_source']
img_path = _source['master-image']['uri']
title = _source['title'].replace(' ',"_").replace("'","").replace("/","-")
img_path = img_path.replace('public://',public_path)
print(img_path)
try:
r=requests.get(img_path,stream=True,headers=headers,timeout=10)
f=open('./img/%s.jpg'%title,'wb')
for a in r.iter_content(chunk_size=1024):
f.write(a)
f.close()
print("Downloaded: "+title)
except:
print("Download failed: "+title)
