使用Python进行上海市租房信息爬取，通过requests + Beautifulsoup对网页内容进行抓取和数据提取。
import requests
from bs4 import BeautifulSoup
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.2669.400 QQBrowser/9.6.10990.400',
'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Encoding':'gzip, deflate, sdch',
'Accept-Language':'zh-CN,zh;q=0.8',
'Connection':'keep-alive',
'Referer':'https://www.baidu.com/link?url=znRkKNPxTXXmN7m4lWy28vWXLWyx_hTEW4bLC5Pi8YR_s5wAdQk4FExi8Fz6HxHc&wd=&eqid=f2ad9a7e00007f1200000004594a3116'
}
urls = [] # 定义空列表，用于创建所有的爬虫链接
#基于for循环，构造完整的爬虫链接
for i in ['zufang']:
url = 'http://sh.lianjia.com/%s/' %i
for j in list(range(1,2075)):
urls.append('http://sh.lianjia.com/%s/d%si1s5' %(i,j)) #拼接所有需要爬虫的链接
#创建csv文件，用于后面的保存数据
file = open('D:/jupyter/lianjia.csv','w')
def readpage(url):
try:
res = requests.get(url)
except ConnectionError as e:
return None
return res
def viewsubway(res2):
try:
view = res2.find_all('span',{'class':'fang-subway-ex'})[0].find_all('span')[0].text
except IndexError as e:
return None
return view
for url in urls: #基于for循环，抓取出所有满足条件的标签和属性列表，存放在find_all中
res = readpage(url)
res = res.text.encode(res.encoding).decode('utf-8')
soup = BeautifulSoup(res,'html.parser')
find_all = soup.find_all(name = 'div',attrs = {'class':'info-panel'})
for i in list(range(len(find_all))): #基于for循环，抓取出所需的各个字段信息
title = find_all[i].find('a')['title'] #每套租房的标语
res2 = find_all[i]
name = res2.find_all('div',{'class':'where'})[0].find_all('span')[0].text #租房所在的小区名称
room_type = res2.find_all('div',{'class':'where'})[0].find_all('span')[1].text #租房的户型
size = res2.find_all('div',{'class':'where'})[0].find_all('span')[2].text[:-3]#租房的面积
#采用列表解析式，删除字符串的首位空格
info = [i.strip() for i in res2.find_all('div',{'class':'con'})[0].text.split('\n')]
region = info[1] #租房所在的区
area = info[2] #租房所在的街道地域
louceng = info[4][:2] #租房所在的楼层
chaoxiang = info[7][3:] #租房的朝向
#每套租房的租金
price = res2.find_all('div',{'class':'price'})[0].find_all('span')[0].text
#地铁站及地铁
view = viewsubway(res2)
if view != None:
subway = view[2:5] #租房附近的地铁线
distance = view[-4:-1] #租房到地铁站的距离
else:
subway = '\0'
distance = '\0'
#将上面的各字段信息值写入并保存到csv文件中
file.write(','.join((name.encode('gbk','ignore'),room_type.encode('gbk','ignore'),size.encode('gbk','ignore'),region.encode('gbk','ignore'),area.encode('gbk','ignore'),louceng.encode('gbk','ignore'),chaoxiang.encode('gbk','ignore'),price.encode('gbk','ignore'),subway.encode('gbk','ignore'),distance.encode('gbk','ignore'),title.encode('gbk','ignore'))) + '\n')
关闭文件(否则数据不会写入到csv文件中)
file.close()```