Python--爬虫数据解析（python爬虫数据解析库不包括）

from lxml import etreehtml = etree.parse('hello.html')result = html.xpath('//li/a[@href="link1.html"]')print result运行结果[]

4.获取最后一个 ​​

from lxml import etreehtml = etree.parse('hello.html')result = html.xpath('//li[last()]/a/@href')# 谓语 [last()] 可以找到最后一个元素print result运行结果['link5.html']

xpath 爬虫示例

???? 示例1

#爬取糗事百科中段子内容和作者名称import requestsfrom lxml import etreeheaders = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.106 Safari/537.36'}url = '= requests.get(url=url, headers=headers).text#解析内容tree = etree.HTML(page_text)div_list = tree.xpath('//div[@class="col1 old-style-col1"]/div')for div in div_list: author = div.xpath('./div[1]/a[2]/h2/text()')[0] #实现局部解析 content = div.xpath('./a[1]/div/span//text()') content = ''.join(content) print(author, content)

???? 示例2

# 联系xpath和解决中文乱码问题dirName = './meinvLibs'if not os.path.exists(dirName): os.mkdir(dirName)url = 'page in range(1,11): if page == 1: new_url = ' else: new_url = format(url%page) page_text = requests.get(url=new_url, headers=headers).text tree = etree.HTML(page_text) a_list = tree.xpath('//div[@class="slist"]/ul/li/a') for a in a_list: img_src = '+ a.xpath('./img/@src')[0] img_name = a.xpath('./b/text()')[0] img_name = img_name.encode('iso-8859-1').decode('gbk') img_data = requests.get(url=img_src, headers=headers).content imgPath = dirName+'/'+img_name+'.jpg' with open(imgPath, 'wb') as fp: fp.write(img_data) print(img_name, "下载成功！！！")

???? 示例3

# 爬取简历模板前五页 通过xpath解析import requests, osfrom lxml import etreesave_resume_dir = './jianli'if not os.path.exists(save_resume_dir): os.mkdir(save_resume_dir)headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.106 Safari/537.36'}url = 'page in range(1,6): if page == 1: new_url = ' else: new_url = format(url%page) page_text = requests.get(url=new_url, headers=headers).text #向每一页发起请求 tree = etree.HTML(page_text) a_list = tree.xpath('//div[@id="container"]/div/a') #获取每一页中所有的简历 for a in a_list: resume_url = a.xpath('./@href')[0] #得到每一份简历的url地址 resume_anme = a.xpath('./img/@alt')[0] #得到每一份简历的名字 resume_anme = resume_anme.encode('iso-8859-1').decode('utf-8') #解决简历名字乱码 resume_text = requests.get(url=resume_url, headers=headers).text #向每一份简历的页面发起请求 tree = etree.HTML(resume_text) resume_down_url = tree.xpath('//div[@class="down_wrap"]/div[2]/ul/li[1]/a/@href')[0] #对每一份简历页面请求数据进行解析，得到下载的url resume_down_text = requests.get(url=resume_down_url, headers=headers).content #下载每一份简历 resume_path = save_resume_dir + '/' + resume_anme + '.rar' with open(resume_path, 'wb') as fp: fp.write(resume_down_text) print(resume_anme+'简历模板下载成功！！！')

???? 示例4

# 爬取梨视频任意板块最热视频 requestsfrom lxml import etreeimport reheaders = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.106 Safari/537.36'}main_url = '= requests.get(url=main_url, headers=headers).texttree = etree.HTML(main_text)div_list = tree.xpath('//*[@id="listvideoListUl"]/li')for li in div_list: detile_url = '+ li.xpath('./div/a/@href')[0] #获取视频详情页 title = li.xpath('./div/a/div[2]/text()')[0] + '.mp4' #获取视频名字 detile_page_text = requests.get(url=detile_url, headers=headers).text #向视频详情页发起请求 ex = 'srcUrl="(.*?)",vdoUrl' #通过正则获取js动态加载的数据 video_url = re.findall(ex, detile_page_text, re.S)[0] #解析出视频url的地址 video_data = requests.get(video_url, headers=headers).content #对视频url发起请求进行下载 with open(f'./video/{title}', 'wb') as fp: fp.write(video_data)

人生是条无名的河，是浅是深都要过； 人生是杯无色的茶，是苦是甜都要喝； 人生是首无畏的歌，是高是低都要唱。