scrapy文件下载(高新技术企业认定网)

# -*- coding: utf-8 -*-
import scrapy
from ..items import XxItem


class GxSpider(scrapy.Spider):
    name = 'gx'
    allowed_domains = ['innocom.gov.cn']
    t_pages = 14
    start_urls = ['http://www.innocom.gov.cn/gxjsqyrdw/gswj/list{}.shtml'.format('' if page ==1 else '_'+str(page)) for page in range(1, t_pages+1)]

    def parse(self, response):
        url_list = response.xpath('//div[@class="list7"]//ul//a/@href').extract()
        for url in url_list:
            yield scrapy.Request(response.urljoin(url), callback=self.parse_detail)

    def parse_detail(self, response):
        a_list = response.xpath('//div[@id="content"]//a')
        for a in a_list:
            title = a.xpath('string(.)').extract_first()
            if '更名' in title or '取消' in title or '变更' in title:    # 更名、取消公告中内容是图片，不处理更名公告
                continue
            url = a.xpath('@href').extract_first()

            item = XxItem()
            item['file_url'] = response.urljoin(url)   # 文件下载地址
            item['file_name'] = title    # 保存的文件名称

            yield item

import scrapy

class XxItem(scrapy.Item):
    file_url = scrapy.Field()
    file_name = scrapy.Field()

from scrapy import Request
from scrapy.pipelines.files import FilesPipeline

class GxPipeline(FilesPipeline):
    def get_media_requests(self, item, info):
        if item['FILENETURL'] == '':
            return None

        # 如果url是列表需要循环发出请求
        yield Request(item['file_url'], meta={'name':item['file_name']})

    def file_path(self, request, response=None, info=None):
        file_name = request.meta['name']     # 重命名文件名称，也可以加入子目录
        return file_name +'.' + request.url.split('.')[-1]

    def item_completed(self, results, item, info):
        file_paths = [x['path'] for ok, x in results if ok]
        if not file_paths :
            with open('download-fail-url.txt', 'aw') as f:
                f.write(item['file_name'] + '---' + item['file_url'] + '\n')
        return item

ITEM_PIPELINES = {
   'zx_gxjsqy.pipelines.GxPipeline': 300,
}

# 设置文件存储目录
FILES_STORE  = r'files'