【原创】Python 二手车之家车辆档案数据爬虫

【原创】Python 二手车之家车辆档案数据爬虫

二手车之家车辆档案数据爬虫

先上效果图

环境

win10 python3.9 lxml、retrying、requests

需求分析

获取车辆信息列表

def _get_car_list(self, _url: str): """ 获取二手车信息列表 """ res = self._parse_url(_url=_url) ret = res.text # 解析获得字符串类型数据 result = etree.HTML(ret) # 转换数据类型为HTML,方便使用xpath url_list = result.xpath('//*[@id="goodStartSolrQuotePriceCore0"]/ul/li/a/@href') if not url_list: print('获取完成！') return for i in url_list: # 有些车型url直接是带域名的 if 'che168.com/' in i: yield '+ i[2:] else: yield '+ i

获取车辆详情信息

def _get_car_info(self, _url: str): """ 获取车辆详情信息 """ res = self._parse_url(_url=_url) ret = res.text # 解析获得字符串类型数据 result = etree.HTML(ret) # 转换数据类型为HTML,方便使用xpath # 标题 title = result.xpath('//div[@class="car-box"]/h3//text()') title = title[1].strip() if len(title) > 1 else title[0].strip() # 上牌时间 play_time = result.xpath('//*[@id="nav1"]/div[1]/ul[1]/li[1]/text()') play_time = play_time[0].strip() if play_time else '-' # 表显里程 display_mileage = result.xpath('//*[@id="nav1"]/div[1]/ul[1]/li[2]/text()') display_mileage = display_mileage[0].strip() if display_mileage else '-' # 变速箱 gearbox = result.xpath('//*[@id="nav1"]/div[1]/ul[1]/li[3]/text()') gearbox = gearbox[0].strip() if gearbox else '-' # 排放标准 emission_standards = result.xpath('//*[@id="nav1"]/div[1]/ul[1]/li[4]/text()') emission_standards = emission_standards[0].strip() if emission_standards else '-' # 排量 displacement = result.xpath('//*[@id="nav1"]/div[1]/ul[1]/li[5]/text()') displacement = displacement[0].strip() if displacement else '-' # 发布时间 release_time = result.xpath('//*[@id="nav1"]/div[1]/ul[1]/li[6]/text()') release_time = release_time[0].strip() if release_time else '-' # 年检到期 annual_inspection_expires = result.xpath('//*[@id="nav1"]/div[1]/ul[2]/li[1]/text()') annual_inspection_expires = annual_inspection_expires[0].strip() if annual_inspection_expires else '-' # 保险到期 insurance_expires = result.xpath('//*[@id="nav1"]/div[1]/ul[2]/li[2]/text()') insurance_expires = insurance_expires[0].strip() if insurance_expires else '-' # 质保到期 warranty_expires = result.xpath('//*[@id="nav1"]/div[1]/ul[2]/li[3]/text()') warranty_expires = warranty_expires[0].strip() if warranty_expires else '-' # 过户次数 number_of_transfers = result.xpath('//*[@id="nav1"]/div[1]/ul[2]/li[5]/text()') number_of_transfers = number_of_transfers[0].strip() if number_of_transfers else '-' # 所在地 location = result.xpath('//*[@id="nav1"]/div[1]/ul[2]/li[6]/text()') location = location[0].strip() if location else '-' # 发动机 engine = result.xpath('//*[@id="nav1"]/div[1]/ul[3]/li[1]/text()') engine = engine[0].strip() if engine else '-' # 车辆级别 vehicle = result.xpath('//*[@id="nav1"]/div[1]/ul[3]/li[2]/text()') vehicle = vehicle[0].strip() if vehicle else '-' # 车身颜色 car_color = result.xpath('//*[@id="nav1"]/div[1]/ul[3]/li[3]/text()') car_color = car_color[0].strip() if car_color else '-' # 燃油标号 fuel_label = result.xpath('//*[@id="nav1"]/div[1]/ul[3]/li[4]/text()') fuel_label = fuel_label[0].strip() if fuel_label else '-' # 驱动方式 drive_mode = result.xpath('//*[@id="nav1"]/div[1]/ul[3]/li[5]/text()') drive_mode = drive_mode[0].strip() if drive_mode else '-' data = [[title, play_time, display_mileage, gearbox, emission_standards, displacement, release_time, annual_inspection_expires, insurance_expires, warranty_expires, number_of_transfers, location, engine, vehicle, car_color, fuel_label, drive_mode, _url]] print(data) self._save_csv(data=data)

版权声明：本文内容由网络用户投稿，版权归原作者所有，本站不拥有其著作权，亦不承担相应法律责任。如果您发现本站中有涉嫌抄袭或描述失实的内容，请联系我们jiasou666@gmail.com 处理，核实后本网站将在24小时内删除侵权内容。