设计思路
1.从host页面获取赶集网所有城市的二手市场marketurl
2.根据marketurl获取每个板块的blockurl
3.通过分析各板块页脚页码元素，分析每个板块有多少个列表页，得到listurl
ps：由于赶集网列表页设置非常混乱（如不显示页数控件，只有一页却可以访问任意页码等等），此功能并未实现，程序只抓取了每个板块的第一页，即blockurl=listurl
4.分析出listurl中的所有详情页detailurl
5.解析出详情页面中的信息，区分转转和赶集商品，获取商品信息
ps：所有信息均入mongodb表中
代码
main.py
#coding=utf-8
'''爬取赶集网所有城市二手市场所有类目的商品信息，由于赶集网列表页数比较混乱，仅抓取各板块下第一页
爬取类目下所有帖子,信息包括:商品标题,发帖时间, 类型,价格,交易地点,新旧程度等
多进程方式爬取
'''
from configparser import ConfigParser
from pymongo import MongoClient
from ganji_crawler import get_citys_market,get_block_urls,get_detail_page_urls,getinfo_from_detailpage
cf = ConfigParser() # 创建conf文件解析对象
cf.read("ganji.conf") # 读取conf文件
host = cf.get('mongodb', 'db_host')
port = cf.getint('mongodb', 'db_port')
client = MongoClient(host, port) # 连接mongodb，创建客户端
ganji_market = client[cf.get('databases', 'db_market')] # 连接二手市场数据库
# 创建collections实例
citys_market = ganji_market[cf.get('collections', 'collection_citys')] # 存放城市url表
city_market_block_url = ganji_market[cf.get('collections', 'collection_block_url')] # 各城市板块url表
market_detailpage_url = ganji_market[cf.get('collections', 'collection_detail_url')] # 详情页url
market_goods_infos = ganji_market[cf.get('collections', 'collection_goods_info')]
#############代码执行部分##############
get_citys_market(host,citys_market) #入库城市及二手市场url
for city in citys_market.find():
get_block_urls(city['link'],city_market_block_url) #入库板块url
for block in city_market_block_url.find({}, {'link': 1, '_id': 0}):
get_detail_page_urls(block['link'],market_detailpage_url) #入库详情页url
for idx, detail in enumerate(market_detailpage_url.find({}, {'link': 1, '_id': 0})): # 入库商品信息
getinfo_from_detailpage(detail['link'], market_goods_infos)
if idx % 1000 == 0:
print('{} records has been inserted ! '.format(idx))
crawler.py
#coding=utf-8
'''爬取赶集网二手市场数据'''
from bs4 import BeautifulSoup
from pymongo import MongoClient,errors
import requests,re,time
def get_citys_market(host,collection):
resp = requests.get(host)
soup = BeautifulSoup(resp.content,'lxml')
links = soup.select('div.all-city > dl > dd > a ') #获取城市列表中所有超链接
for link in links:
collection.insert_one({
'link' : link['href']+'wu/',
'city' : link.string
})
def get_block_urls(city_market_url,collection):
'''从一个城市的二手市场页面抓取所有区块url
city_market_url:某个城市的二手市场url
collection：解析出的板块url存入的数据表'''
resp = requests.get(city_market_url)
soup = BeautifulSoup(resp.content,'lxml')
try:
div_navigate = soup.select('div.main')[0]
except IndexError:
return
for a in div_navigate.select('a'):
try:
href = a['href']
if href.startswith('/'): #清洗脏数据，全部分类中有#开头的
collection.insert_one({'link': city_market_url[:-4] + href}) #拼拼凑板块url，这里因为赶集网设置比较特殊，需要去除一些无效字符
except errors.DuplicateKeyError as e:
print(e)
def get_detail_page_urls(blockurl,collection):
'''赶集网板块的页数判断和访问就是个坑啊，完全没有判断板块下有多少页的规律，这里先提取各版块第一页的详情页url'''
resp = myRequestGet(blockurl)
if not resp:
return
soup = BeautifulSoup(resp.content,'lxml')
try:
layoutlist = soup.select('dl.list-bigpic.clearfix') #定位到每条数据dl标签上
except IndexError:
return
time.sleep(1)
for layout in layoutlist:
links = layout.select('a') #获取此标签下所有超链
for link in links:
href = link['href']
if href.startswith('http://m.zhuanzhuan.58.com'): #筛选出转转数据url
#由于获取的转转url是通过js获取商品信息的，所以需要改一下url形式，以便css path可以找到目标信息
infoId = re.findall(r'infoId=(\d+)&', href)[0]
href = 'http://zhuanzhuan.58.com/detail/{}z.shtml'.format(infoId)
try:
collection.insert_one({'source':'zhuanzhuan','link': href}) #入库转转url-改写后
except errors.DuplicateKeyError as e:
print(e)
elif href.endswith('.htm'):
try:
collection.insert_one({'source':'ganji','link': href}) #入库赶集url
except errors.DuplicateKeyError as e:
print(e)
def getinfo_from_detailpage(detailurl,collection):
'''从详情页获取商品信息'''
resp = myRequestGet(detailurl)
if not resp:
return
soup = BeautifulSoup(resp.content, 'lxml')
time.sleep(0.1)
#转转商品信息获取
if resp.url.startswith('http://zhuanzhuan'):
try:
title = soup.select(' h1.info_titile ')[0].string
price = ''.join(soup.select(' span.price_now ')[0].stripped_strings)
area = soup.select(' div.palce_li > span > i ')[0].string
desc = soup.select(' div.baby_kuang.clearfix > p')[0].string
except IndexError:
return
#入表
collection.insert_one({
'source' : 'zhuanzhuan',
'title': title,
'price': price,
'area': area,
'desc': desc})
else: #赶集商品信息获取
try:
title = soup.select(' h1.title-name ')[0].string
price = soup.select(' i.f22.fc-orange.f-type ')[0].string
area = ''.join(soup.select(' ul.det-infor > li:nth-of-type(3) ')[0].stripped_strings)
desc = soup.select(' .second-sum-cont')[0].get_text().strip()
except IndexError:
return
collection.insert_one({
'source': 'ganji',
'title': title,
'price': price,
'area': area,
'desc': desc
})
def myRequestGet(url):
'''会遇到被封的情况，在这里把requests.get包裹一层函数，如果异常，则sleep(10)'''
try:
resp = requests.get(url)
return resp
except requests.exceptions.RequestException as e:
print('Requests Error -----------{}-----------wait 10 seconds'.format(str(e.__class__)))
time.sleep(10)
return None
except Exception as e:
print('Other Eroor -----------{}-----------wait 10 seconds'.format(str(e.__class__)))
time.sleep(10)
return None
#判断页面是否存在，由于判断时需要读取页面并生成soup后判断，所以干脆传入soup对象，而不是url
def exists(soup):
if soup.title.string == '您访问的网页不存在':
return False
else:
return True
if __name__ == '__main__':
##############一些初始化##############
client = MongoClient('mongodb://localhost:27017')
ganji_market = client['ganji_market'] #赶集网二手市场数据库
host = 'http://www.ganji.com/index.htm'
#创建存储城市及其二手市场url的表
citys_market = ganji_market['citys_market']
#创建存储各城市二手市场板块url的collections，并将link字段设置为唯一索引，避免出现重复的link
city_market_block_url = ganji_market['city_market_block_url']
city_market_block_url.ensure_index('link', unique=True)
#创建详情页url存储表
market_detailpage_url = ganji_market['market_detailpage_url']
market_detailpage_url.ensure_index('link', unique=True)
#商品信息入库
market_goods_infos = ganji_market['market_goods_infos']
##############代码执行部分##############
# get_citys_market(host,citys_market) #入库城市及二手市场url
# get_citys_market(host)
# for city in citys_market.find():
# get_block_urls(city['link'],city_market_block_url) #入库板块url
# get_block_urls('http://xa.ganji.com/wu/')
# for block in city_market_block_url.find({}, {'link': 1, '_id': 0}):
# get_detail_page_urls(block['link'],market_detailpage_url) #入库详情页url
# get_detail_page_urls('http://xa.ganji.com/ershoubijibendiannao/')
for idx,detail in enumerate(market_detailpage_url.find({}, {'link': 1, '_id': 0}).skip(12000)): #入库商品信息
getinfo_from_detailpage(detail['link'],market_goods_infos)
if idx % 1000 == 0:
print ('{} records has been inserted ! '.format(idx))
# getinfo_from_detailpage('http://zhuanzhuan.58.com/detail/755842657703362564z.shtml')
# getinfo_from_detailpage('http://xa.ganji.com/ershoubijibendiannao/2266604261x.htm')
ganji.conf
[mongodb]
db_host = localhost
db_port = 27017
[databases]
db_market = ganji_market
[collections]
collection_citys = citys_market
collection_block_url = city_market_block_url
collection_detail_url = market_detailpage_url
collection_goods_info = market_goods_infos
总结
1.没有使用多进程，是因为函数都要传入两个参数（包括一个collection对象），暂且先不修改函数适用map方式了。multiprocessing和threading模块还需要再学习一下
2.为了增强程序健壮性，执行过程中增加了一些try模块，在遇到爬虫被封的时候停顿10s再接着抓取，目前看效果不错
3.抓取是按照模板设计的，通用性的问题依然存在，不知是否可以使用scrapy解决
4.引入了一个configparser模块来创建配置文件读取相关信息，以后可以继续使用
5.由于爬虫被封，商品详情url仅抓取了20w，因此商品信息也仅能解析出这么多。后增加了程序的健壮性，未重新运行。
