关于xpath的具体教程可以看崔大神的这篇:https://cuiqingcai.com/2621.html
- 安装lxml:pip install lxml
- 路径表达式
表达式 | 描述 |
---|---|
nodename | 选取此节点的所有子节点 |
/ | 从根节点选取 |
// | 从匹配选择的当前节点选择文档中的节点,而不考虑它们的位置 |
. | 选取当前节点 |
.. | 选取当前节点的父节点 |
@ | 选取属性 |
实例代码:
from lxml import etree
text = """
<html lang="en">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=7,9,10,11"/>
<meta name="renderer" content="webkit">
<title>Title</title>
</head>
<body>
<div class="catalog-list column-3">
<ol>
<li class="level1">
<span class="index">1</span>
<span class="text"><a href="#1">简介</a></span>
</li>
<li class="level1">
<span class="index">2</span>
<span class="text"><a href="#2">语法</a></span>
</li>
<li class="level1">
<span class="index">3</span>
<span class="text"><a href="#3">表达式</a></span>
</li>
<li class="level2">
<span class="index">4</span>
<span class="text"><a href="#4">运算符</a></span>
</li>
</ol><ol><li class="level1">
<span class="index">5</span>
<span class="text"><a href="#5">标准函数</a></span>
</li>
<li class="level3">
<span class="index">6</span>
<span class="text"><a href="#6">使用</a></span>
</li>
<li class="level1">
<span class="index">7</span>
<span class="text"><a href="#7">标准</a></span>
</li>
<li class="level1">
<span class="index">8</span>
<span class="text"><a href="#8">轴定义</a></span>
</li>
</ol><ol><li class="level4">
<span class="index">9</span>
<span class="text"><a href="#9">节点关系</a></span>
</li>
<li class="level4">
<span class="index">10</span>
<span class="text"><a href="#10">存取函数</a></span>
</li>
<li class="level4">
<span class="index">11</span>
<span class="text"><a href="#11">语言升级</a></span>
</li>
<li class="level4">
<span class="index">12</span>
<span class="text"><a href="#12">实例</a></span>
</li>
</ol>
<ol>
<li>A</li>
<li>A</li>
<li>A</li>
</ol>
</div>
</body>
</html>
"""
# 数据初始化
data = etree.HTML(text)
# 打印初始化的数据
result = etree.tostring(data)
print(result)
# 根据标签名称选取此节点,如body等
result = data.xpath('body')
# / 从根节点选取
result = data.xpath('/html/head/meta')
# // 从匹配选择的当前节点选择文档中的节点,而不考虑它们的位置。
# 获取所有的<ol>标签
result = data.xpath('//ol')
# . 选取当前节点
# 选取当前result下的<li>标签
li = result[0].xpath('./li')
# @ 选取属性
# 获取 <li> 标签的所有 class
result = data.xpath('//li/@class')
print(result)
# 获取所有<a>标签的href值
result = data.xpath('//a/@href')
print(result)
# 获取 <li> 标签下 href 为 #7 的 <a> 标签
result = data.xpath('//li//a[@href="#7"]')
# 获取 <li> 标签下的所有 <a> 标签
result = data.xpath('//li//a') #<a> 并不是 <li> 的子元素,所以,要用双斜杠
CSDN实例
import requests
from lxml import etree
import json
# 爬取CSDN论坛信息的类
class BBSpider:
url = "https://bbs.csdn.net/tech_hot_topics"
headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36'}
# 存放最终的数据,用于数据存储
result_list = []
# 根据给到的url获取网页数据并返回
def get_data(self, url):
r = requests.get(url=url, headers=self.headers)
data = r.text
return data
# 解析得到的网页数据,分析出该页的帖子的时间、帖子标题、帖子链接
def parse_data(self, data, url):
# 将网页的数据进行转化
html = etree.HTML(data)
if data is not None:
# 帖子内容都在class="list_1"的div标签中,在无序列表ul,以li标签的形式进行展示
div = html.xpath('//div[@class="list_1"]//li')
# 遍历每个li标签,获取li标签中包含的时间time、帖子标题content、帖子链接link
for tiezi in div:
dict = {}
# 获取li标签中包含的时间time
dict['time'] = tiezi.xpath('./span/text()')[0]
# 获取li标签中包含的帖子标题content
dict['content'] = tiezi.xpath('./a/text()')[0]
# 获取li标签中包含的帖子链接link
dict['link'] = 'https://bbs.csdn.net' + tiezi.xpath('./a/@href')[0]
# 将获取到的帖子信息存放到类成员变量result_list中,便于后续的数据保存
self.result_list.append(dict)
else:
print("NO Data: "+url)
# 存储数据,将result_list中的数据以json的格式保存到data.json文件中
def save_data(self):
data = json.dumps(self.result_list)
with open("data.json", "w") as f:
f.write(data)
# 由于CSDN的论坛是以翻页的形式进行展示的,翻页的链接为:域名+"?page=xxx",以下是对每一页的数据进行数据的获取、解析,最后保存的操作
def run(self):
# 获取从第1页到第9页的数据
for i in range(1,10):
url = self.url + "?page="+str(i)
data = self.get_data(url)
self.parse_data(data, url)
print(len(self.result_list))
# 保存第1页到第9页全部的网页数据
self.save_data()
# 对CSDN的论进行网页爬取
spider = BBSpider()
spider.run()
运行结果
生成了一个data.json文件,文件内容如下:
[{"content": "\u6709\u4e2a\u81ea\u8eab\u8fed\u4ee3\u51fa\u73b0\u4e86\u95ee\u9898\uff0c\u65e0\u6cd5\u7406\u89e3\u554a", "link": "https://bbs.csdn.net/topics/392468373", "time": "2018-10-27 12:10:41"}, {"content": "\u8fde\u7eed\u5c1d\u8bd56\u5c0f\u65f6\u65e0\u679c \u80fd\u89e3\u51b3\u8fd9\u4e2a\u95ee\u9898\u7684\u4eba\u53ef\u80fd\u4e0d\u8d85\u8fc73\u4e2a", "link": "https://bbs.csdn.net/topics/392468284", "time": "2018-10-27 02:44:34"}, {"content": "Http\u8bf7\u6c42header\u4e2d\u81ea\u5b9a\u4e49\u7684\u53c2\u6570\u662f\u5982\u4f55\u4f20\u5230PHP SERVER\u53d8\u91cf\u4e2d\u7684", "link": "https://bbs.csdn.net/topics/392468081", "time": "2018-10-26 16:39:40"}, {"content": "\u6709\u73b0\u6210winform\u57fa\u4e8e.net FX2.0\u7684\uff0c\u73b0\u5728\u7528\u8f6c\u79fb\u5230LINUX\u4e0a", "link": "https://bbs.csdn.net/topics/392468065", "time": "2018-10-26 16:25:51"}······]
以下为相关的解析:
具体代码链接:https://github.com/zhuyecao/kaikeba/tree/master/kaikeba/xpath