由于网页版的微博存在滚动刷新的特性，使用传统的获取html再解析的方法不再可行，而移动版的微博每页微博条数是固定的，不存在网页版滚动刷新的现象，而且URL设计更加简单，通过修改URL重复模拟登录即可实现翻页效果。本文最终实现了三个函数，分别是爬取博主主页信息、爬取关注人信息、爬取原创微博信息。

首先登录 https://weibo.cn，在登录时页面可能一直不响应，我也遇到了相同的问题，我是用第三方软件登录然后绑定到我的微博实现了登录，登录的目的是获取cookie，然后使用这个cookie模拟登录微博，爬取微博内容。

获取cookie的方式：

step1:登录微博移动版
step2:打开“检查”工具 ->右键重新加载->Network->Request Headers->Cookie
注意：cookie的时效不超过24小时，所以每次运行爬虫需要更新

getcookie.jpg

URL分析：

以 @追风少年刘全有 的微博为例
在移动版微博中，博主的主页URL是 https://weibo.cn/u/2150511032，同时使用https://weibo.cn/2150511032也可以访问，其中数字串是唯一标识博主的user_id；
刘全有关注的人网页的URL为https://weibo.cn/2150511032/follow?page=1，也就是主页URL拼接'/follow?page=x',给page赋不同值即可实现翻页；

follow.png

话不多说上代码：

from bs4 import BeautifulSoup
import requests
from lxml import etree
import re
import pymysql.cursors

cookie = {"Cookie": "yourCookie"}

#获取博主关注的人的urls
def get_guanzhu_urls(url):

    url_g = url+ "/follow?page="
    count="1"
    url_init=url_g+count
    html = requests.get(url_init, cookies=cookie, verify=False).content #cookie登录
    selector = etree.HTML(html)
    pageNum = (int)(selector.xpath('//input[@name="mp"]')[0].attrib['value']) #获取关注人页数

    for i in range(1,pageNum+1):
        url_new=url_g+ (str)(i)
        html = requests.get(url_new, cookies=cookie, verify=False).content
        soup = BeautifulSoup(html, 'html5lib')
        person=soup.find_all('td')

        in_val=[]

        try:
            conn = pymysql.connect(host='localhost', user='root', passwd='xxx', db='weibo_spider', charset='utf8mb4')
            with conn.cursor() as cursor:
                for i in range(0, len(person)):
                    if (i + 1) % 2 == 0:
                        user = (person[i].a.string , person[i].a.get("href"), url)
                        in_val.append(user)

                sql = "insert into `gurls`(`weiboid`,`url`,`prewid`) VALUES(%s,%s,%s)"
                cursor.executemany(sql,in_val)#将数据批量导入数据库
                conn.commit()
        finally:
            conn.close()

#获取博主个人信息：ID、粉丝数、关注数、关注列表URL
def get_host_user_info(url):

    html = requests.get(url, cookies=cookie, verify=False).content
    soup = BeautifulSoup(html,'html5lib')
    weiboid=soup.find_all('span',class_='ctt')[0].get_text().split()[0] #span标签
    myurl=url

    info=soup.find('div', class_='tip2')
    infolist=info.find_all('a')
    guanzhu_url = infolist[0].get("href")
    pattern_num = re.compile(r'.*\[(.*)\]')
    guanzhu_num = re.findall(pattern_num,(infolist[0].string))[0]
    fan_num = re.findall(pattern_num, (infolist[1].string))[0]

    print(weiboid,myurl,guanzhu_url,guanzhu_num,fan_num)

    try:
        conn = pymysql.connect(host='localhost', user='root', passwd='xxx', db='weibo_spider', charset='utf8mb4')
        with conn.cursor() as cursor:
            sql="insert into `wusers`(`weiboid`,`myurl`,`follower_num`,`guanzhu_num`,`guanzhu_url`) VALUES (%s,%s,%s,%s,%s)"
            cursor.execute(sql,(weiboid,myurl,fan_num,guanzhu_num,guanzhu_url))
            conn.commit()
    finally:
        conn.close()

#获取博主微博博文
def get_weibo_contents(url):

    #只爬取原创微博
    html = requests.get(url, cookies=cookie, verify=False).content
    selector = etree.HTML(html)
    pageNum = (int)(selector.xpath('//input[@name="mp"]')[0].attrib['value'])
    soup=BeautifulSoup(html, 'html5lib')
    weiboid=soup.find_all('span',class_='ctt')[0].get_text().split()[0] #span标签
    print(weiboid)

    word_count = 1
    print(pageNum)

    for page in range(1, pageNum + 1):

        # 获取lxml页面
        url_new = url+"?filter=1&page="+(str)(page)
        print(url_new)
        lxml = requests.get(url_new, cookies=cookie, verify=False).content
        soup = BeautifulSoup(lxml, 'html5lib')


        content=soup.find_all('span',class_="ctt")
        comment=soup.find_all('a',class_="cc")

        in_weibo=[]
        try:
            conn = pymysql.connect(host='localhost', user='root', passwd='xxx', db='weibo_spider', charset='utf8mb4')
            with conn.cursor() as cursor:
                for (con,com) in zip(content,comment):
                    pattern_num = re.compile(r'.*\[(.*)\]')
                    com_num = re.findall(pattern_num, (com.string))[0]
                    weibo=(weiboid,con.get_text(),com_num)
                    in_weibo.append(weibo)

                sql = "insert into `weibos`(`weiboid`,`weibo_content`,`comment`) VALUES(%s,%s,%s)"
                cursor.executemany(sql,in_weibo)#将数据批量导入数据库
                conn.commit()
        finally:
            conn.close()


if __name__ == '__main__':
    url = "https://weibo.cn/1496852380"
    #get_host_user_info(url)
    #get_guanzhu_urls(url)
    #get_weibo_contents(url)

我是将爬取的数据存到MySQL数据库中，你也可以用其他方式存储。