使用Python爬取易车网的车型数据

2024年3月2日发(作者：五菱宏光官网商城)

der的全部代码# coding=utf-8import jsonimport reimport scrapyfrom s import Rulefrom tractors import LinkExtractorfrom import BrandItem, SerialItem, ModelItem# json替换keydef replacea(matched): return \'\"\' + (\'value\') + \'\":\'# 解析车系Itemdef parse_serial_item(serial, bid, vendor): item = SerialItem() item[\'id\'] = (\'div/div/a/@id\')[0].re(r\'n(d+)\')[0] item[\'bid\'] = bid item[\'name\'] = (\'div/div/a/@title\')[0].extract() item[\'vendor\'] = vendor item[\'logo\'] = (\'div/div/a/img/@src\')[0].extract() sell = (\'div/ul/li[@class=\"price\"]/a/text()\')[0].re(ur\'停售\') item[\'sell\'] = \'0\' if sell else \'1\' return itemclass YiCheSpider(): name = \"yiche\" rules = ( # 所有车系 Rule(LinkExtractor(allow=(r\'/tree_chexing/mb_d+/$\',)), callback=\'parse_serial\', follow=True), # 在售车型 Rule(LinkExtractor(allow=(r\'/w+/$\',)), callback=\'parse_model\', follow=True), # 停售车型 Rule(LinkExtractor(allow=(r\'/AjaxNew/?csID=d+&year=d+$\',)), callback=\'parse_model_selled\', follow=True), ) def start_requests(self): url = \'/CarInfo/?tagtype=chexing&pagetype=masterbrand&objid=0\' yield t(url, callback=, dont_filter=True) # 解析品牌 def parse(self, response): print \'==> %s\' %

result = (\'(?Pw+):\', replacea, [(\'{\'):(\'}\') + 1]) data = (result) for char in data[\'char\']: print \'==> %s\' % char try: for brand in data[\'brand\'][\'%s\' % char]: item = BrandItem() item[\'id\'] = brand[\'id\'] item[\'name\'] = brand[\'name\'] item[\'logo\'] = \'/bt/car/default/images/logo/masterbrand/png/100/m_%s_\' % item[\'id\'] item[\'initial\'] = char yield item url = \'/tree_chexing/mb_%s/\' % item[\'id\'] request = t(url, callback=_serial, dont_filter=True) [\'bid\'] = item[\'id\'] yield request except KeyError: pass # 解析车系 def parse_serial(self, response): print \'==> %s\' % bid = [\'bid\'] # 品牌下的全部车系及车型 brands = (\'//*[@id=\"divCsLevel_0\"]/*\') size = len(brands) if size % 2 == 0: for i in range(size / 2): vendor = brands[i * 2].xpath(\'a/text()\')[0].re(r\'(w+)>>\')[0] for serial in brands[i * 2 + 1].xpath(\'div\'): item = parse_serial_item(serial, bid, vendor) yield item url = \'%s\' % (\'div/div/a/@href\')[0].extract() request = t(url, callback=_model, dont_filter=True) [\'sid\'] = item[\'id\'] yield request else: for serial in (\'div\'): item = parse_serial_item(serial, bid, \'\') yield item url = \'%s\' % (\'div/div/a/@href\')[0].extract() request = t(url, callback=_model, dont_filter=True) [\'sid\'] = item[\'id\'] yield request # 解析车型 def parse_model(self, response): print \'==> %s\' % sid = [\'sid\'] # 在售车型 classify = \'\' for tr in (\'//*[@id=\"compare_sale\"]/tbody/*\'): tit = (\'@class\') if len(tit) == 0: item = ModelItem() item[\'id\'] = (\'td[1]/a[1]/@href\')[0].re(r\'/m(d+)/\')[0] item[\'sid\'] = sid item[\'name\'] = (\'td[1]/a[1]/text()\')[0].extract() item[\'classify\'] = classify item[\'sell\'] = \'1\' yield item else: ths = (\'th[1]/text()\') th = ths[1].extract().rstrip() if len(ths) > 1 else ths[0].extract().rstrip() strong = (\'th[1]/strong/text()\')[0].extract() classify = strong + ((\'/\' + th) if len(th) > 0 else \'\')

# 停售车型 years = (\'//*[@id=\"carlist_nosaleyear\"]/a/@id\').extract() for year in years: url = \'/AjaxNew/?csID=%s&year=%s\' % (sid, year) request = t(url=url, callback=_model_selled, dont_filter=True) [\'sid\'] = sid yield request @staticmethod def parse_model_selled(response): print \'==> %s\' % sid = [\'sid\'] try: datas = (_as_unicode()) for data in datas: classify = data[\'Engine_Exhaust\'] + \'/\' + data[\'MaxPower\'] + \' \' + data[\'InhaleType\'] for car in data[\'carList\']: item = ModelItem() item[\'id\'] = car[\'CarID\'] item[\'sid\'] = sid item[\'name\'] = car[\'YearType\'] + \' \' + car[\'Name\'] item[\'classify\'] = classify item[\'sell\'] = \'0\' yield item except ValueError: print \'model parse error,serial_id[%s].\' % sid pass