死亡公司公墓

本次爬取的是新经济死亡公司数据库，从死亡原因，获投状态，存活天数等多个指标呈现死亡公司全貌。
使用Scrapy爬虫框架抓取数据。

抓取

1.分析请求

url = "https://www.itjuzi.com/deathCompany"
通过刷新页面可以在网络请求里发现Ajax请求，返回数据格式为Json。
向该响应的Request URL发送请求即可。

2.明确抓取字段

抓取字段如下：

    # 公司id
    com_id = scrapy.Field()
    # 公司名称
    com_name = scrapy.Field()
    # 关闭时间
    com_change_close_date = scrapy.Field()
    # 公司简介
    com_des = scrapy.Field()
    # 行业
    cat_name = scrapy.Field()
    # 地点
    com_prov = scrapy.Field()
    # 获投状态
    com_fund_status_name = scrapy.Field()
    # 成立时间
    born = scrapy.Field()
    # 存活天数
    live_time = scrapy.Field()
    # 团队
    com_team = scrapy.Field()
    # 行业标签
    com_tag = scrapy.Field()
    # 死亡原因
    closure_type = scrapy.Field()

3.spider/解析数据

使用Jsonpath。

class OrangeSpider(scrapy.Spider):
    name = 'orange'
    allowed_domains = ['www.itjuzi.com']
    page = 1
    url = "https://www.itjuzi.com/api/closure?com_prov=&sort=&keyword=&cat_id=&page="
    start_urls = [url+str(page),]

    def parse(self, response):
        data = json.loads(response.text)
        infos = jsonpath.jsonpath(data,"$..info")[0]
        for info in infos:
            item = ItjuziItem()

            # 公司id
            item['com_id'] = jsonpath.jsonpath(info,'$..com_id')[0]
            # 公司名称
            item['com_name'] = jsonpath.jsonpath(info,'$..com_name')[0]
            # 关闭时间
            item['com_change_close_date'] = jsonpath.jsonpath(info,'$..com_change_close_date')[0]
            # 公司简介
            item['com_des'] = jsonpath.jsonpath(info,'$..com_des')[0]
            # 行业
            item['cat_name'] = jsonpath.jsonpath(info,'$..cat_name')[0]
            # 地点
            item['com_prov'] = jsonpath.jsonpath(info,'$..com_prov')[0]
            # 获投状态
            item['com_fund_status_name'] = jsonpath.jsonpath(info,'$..com_fund_status_name')[0]
            # 成立时间
            item['born'] = jsonpath.jsonpath(info,'$..born')[0]
            # 存活天数
            item['live_time'] = jsonpath.jsonpath(info,'$..live_time')[0]
            # 团队
            item['com_team'] = jsonpath.jsonpath(info,'$..com_team..name')
            # 行业标签
            item['com_tag'] = ",".join(jsonpath.jsonpath(info,'$..com_tag..tag_name'))
            # 死亡原因
            item['closure_type'] = jsonpath.jsonpath(info,'$..closure_type..name')
            yield item
        self.page += 1
        yield scrapy.Request(self.url+str(self.page),callback=self.parse)

Jsonpath返回一个列表。

4.middlewares

爬取数据过多会封禁IP，所以在下载中间件里加上IP代理。

class ItjuziDownloaderMiddleware(object):
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the downloader middleware does not modify the
    # passed objects.
    def __init__(self):
        self.user_agents = [
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134',
            'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0) Gecko/20100101 Firefox/6.0',
            'Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11',
            'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko',
            'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)'
        ]
    def proxy(self):
        # 代理服务器
        proxyHost = "http-dyn.abuyun.com"
        proxyPort = "9020"

        # 代理隧道验证信息
        proxyUser = "H262AH098992ITYD"
        proxyPass = "18C9DE75BD618524"

        proxyMeta = "http://%(user)s:%(pass)s@%(host)s:%(port)s" % {
            "host": proxyHost,
            "port": proxyPort,
            "user": proxyUser,
            "pass": proxyPass,
        }

        # proxies = {
        #     "http":proxyMeta,
        #     "https":proxyMeta,
        # }
        # 这里不要加 HTTP
        proxies = proxyMeta
        return proxies

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        """在请求发送之前处理请求"""
        # 随机的User-Agent
        request.headers['User-Agent'] = random.choice(self.user_agents)
        # 设置代理
        request.meta['proxy'] = self.proxy()
        # 取消SSL验证
        request.meta['verify']= False
        
        return None

    def process_response(self, request, response, spider):
        # Called with the response returned from the downloader.
        # Must either;
        # - return a Response object
        # - return a Request object
        # - or raise IgnoreRequest
        return response

    def process_exception(self, request, exception, spider):
        # Called when a download handler or a process_request()
        # (from other downloader middleware) raises an exception.

        # Must either:
        # - return None: continue processing this exception
        # - return a Response object: stops process_exception() chain
        # - return a Request object: stops process_exception() chain
        # return self._retry(request, exception, spider)
        pass

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)