Python: 使用Scrapy定制可动态配置的爬虫

from  sqlalchemy import  Column, String , DateTime, Integer from  sqlalchemy.ext.declarative import  declarative_base  Base = declarative_base()  class  Rule (Base) :      __tablename__ = 'rules'       id = Column(Integer, primary_key=True )     name = Column(String)     allow_domains = Column(String)     start_urls = Column(String)     next_page = Column(String)     allow_url = Column(String)     extract_from = Column(String)     title_xpath = Column(String)     body_xpath = Column(String)     publish_time_xpath = Column(String)     source_site_xpath = Column(String)     enable = Column(Integer)  

# -*- coding: utf-8 -*-  import  scrapy from  scrapy.contrib.spiders import  CrawlSpider, Rule from  scrapy.contrib.linkextractors import  LinkExtractor  class  Article (scrapy.Item) :      title = scrapy.Field()     url = scrapy.Field()     body = scrapy.Field()     publish_time = scrapy.Field()     source_site = scrapy.Field()  class  DeepSpider (CrawlSpider) :      name = "Deep"    def  __init__ (self,rule) :          self.rule = rule         self.name = rule.name         self.allowed_domains = rule.allow_domains.split("," )         self.start_urls = rule.start_urls.split("," )         rule_list = []  #添加`下一页`的规则   if  rule.next_page:             rule_list.append(Rule(LinkExtractor(restrict_xpaths = rule.next_page)))  #添加抽取文章链接的规则          rule_list.append(Rule(LinkExtractor(             allow=[rule.allow_url],             restrict_xpaths = [rule.extract_from]),             callback='parse_item' ))         self.rules = tuple(rule_list)         super(DeepSpider, self).__init__()    def  parse_item (self, response) :          self.log('Hi, this is an article page! %s'  % response.url)          article = Article()          article["url" ] = response.url          title = response.xpath(self.rule.title_xpath).extract()         article["title" ] = title[0 ] if  title else  ""           body = response.xpath(self.rule.body_xpath).extract()         article["body" ] =  '\n' .join(body) if  body else  ""           publish_time = response.xpath(self.rule.publish_time_xpath).extract()         article["publish_time" ] = publish_time[0 ] if  publish_time else  ""           source_site = response.xpath(self.rule.source_site_xpath).extract()         article["source_site" ] = source_site[0 ] if  source_site else  ""    return  article  

# -*- coding: utf-8 -*-  from  spiders.deep_spider import  DeepSpider from  model.config import  DBSession from  model.rule import  Rule  # scrapy api  from  scrapy import  signals, log from  twisted.internet import  reactor from  scrapy.crawler import  Crawler from  scrapy.settings import  Settings   RUNNING_CRAWLERS = []  def  spider_closing (spider) :   """Activates on spider closed signal"""      log.msg("Spider closed: %s"  % spider, level=log.INFO)     RUNNING_CRAWLERS.remove(spider)  if  not  RUNNING_CRAWLERS:         reactor.stop()  log.start(loglevel=log.DEBUG)  settings = Settings()  # crawl settings  settings.set("USER_AGENT" , "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36" )  db = DBSession() rules = db.query(Rule).filter(Rule.enable == 1 ) for  rule in  rules:     crawler = Crawler(settings)     spider = DeepSpider(rule)  # instantiate every spider using rule      RUNNING_CRAWLERS.append(spider)   # stop reactor when spider closes      crawler.signals.connect(spider_closing, signal=signals.spider_closed)     crawler.configure()     crawler.crawl(spider)     crawler.start()  # blocks process so always keep as the last statement  reactor.run()