#import your model here
from scrapy import cmdline
#your class&function here

if __name__ == "__main__":
    cmdline.execute(["scrapy", "crawl", "爬虫名字"])

import os,sys
project_dir=os.path.abspath(os.path.dirname(__file__))#获取了当前文件也就是setting的目录
IMAGES_STORE = os.path.join(project_dir,"images")#设置image的存储文件夹为当前目录下的image

from scrapy import Request
def func(self, response):
  response.meta[k]#这样就和字典一样取出了v
  process...
  yield Request(url='localhost',meta={k:v},callback=self.func)

#文件顶部引入
from engine.items import MyItem
...
item=MyItem()
item['x']=变量a
item['y']=变量b
yield item

def parse_item(self,response):
    item=ItemLoader(item=JianshuItem(),response=response)
    item.add_xpath('title','/html/body/div[1]/div[2]/div[1]/h1/text()')
    item.add_value('likes',re.findall('<script type="application/json" data-name="page-data">(.*?)</script>',response.body.decode('utf-8')))
    item.add_css('author','body > div.note > div.post > div.article > div.author > div > span > a::text')
    item.add_value('noteid',response.url)
    item.add_value('imageurl',response.meta['imageurl'])
    return item.load_item()

def callme(value):
    print("我被调用啦")
    return value

class JianshuItem(scrapy.Item):

    title=scrapy.Field()
    likes=scrapy.Field(
        output_processor=MapCompose(callme))#把这个里面的东西出来
    author=scrapy.Field()
    imageurl=scrapy.Field(
        input_processor=MapCompose(lambda x: "http:"+x.split("?")[0])
    )
    noteid=scrapy.Field()

ITEM_PIPELINES = {
   'aa.pipelines.likesPipeline': 1,
   'aa.pipelines.SqlPipeline': 2,
}

class MyPipline(object):
    process_item(self, item, spider):
      ...
      return item

#piplines文件中引入
from scrapy.pipelines.images import ImagesPipeline

setting中指明两个变量
IMAGES_URLS_FIELD = "imageurl"#哪个字段是image的url
IMAGES_STORE = os.path.join(project_dir,"image")#存在哪个路径下
添加进piplines中
ITEM_PIPELINES = {
'scrapy.pipelines.images.ImagesPipeline':1,#普通的图片pipeline
                  }

class JobboleImagePipline(ImagesPipeline):#自定义的管道进行路径存放
    def item_completed(self, results, item, info):
        for status,p in results:
            try:
                item["imagepath"]=p["path"]
                item["imageurl"]=item["imageurl"][0]
            except TypeError:
                item["imageurl"]="None"
                item["imagepath"] = "None"
        return item

class SqlPipeline(object):#构造sql语句并执行即可
    def process_item(self,item,spider):
        '''item={'author': ['城北听雪'],
        'imageurl': ['http://upload-images.jianshu.io/upload_images/14074951-f700da6a9b168924.jpg'],
        'likes': ['33'],
        'noteid': ['https://www.jianshu.com/p/3e652aafb69b'],
        'title': ['5天流浪加德满都，给你看一个不一样的真实尼泊尔']}'''
        a={ k:v[0] for k,v in item.items()}#构造字符串
        str1 = ",".join([i for i in a.keys()])
        str2 = "','".join([i for i in a.values()])
        sql="INSERT INTO jianshu ({0}) VALUES ('{1}')".format(str1,str2)#构造sql
        conn=pymysql.connect(host='127.0.0.1',port=3306,user='root',db='jobol',password='不告诉你',charset='utf8mb4')
        cur=conn.cursor()
        try:
            cur.execute(sql)
            conn.commit()
        except Exception as e:
            conn.rollback()
            print(e)
        finally:
            cur.close()
            conn.close()
        return item

import pymysql
from twisted.enterprise import adbapi
import pymysql.cursors
#先引入

class asyPipeline(object):
    def __init__(self,dbpool):#默认传入一个连接池
        self.dbpool=dbpool

    @classmethod#声明一个类方法
    def from_settings(cls,settings):
    #注意是settings有S这个类方法和实例没有关系,接受一个setting也就是我们的配置文件
    #把变量拿过来，传入adbpool里面定义一个dbpool
        dbpool=adbapi.ConnectionPool('pymysql',
                                     host=settings['MYSQL_HOST'],
                                     db=settings['DB_NAME'],
                                     user=settings['MYSQL_USER'],
                                     password=settings['MYSQL_PASSWOED'],
                                     charset=settings['CHARSET'],
                                     cursorclass=pymysql.cursors.DictCursor,
                                     use_unicode=True)
    #这里可以写死也可以用这个方法读取setting里面的配置信息(建议)
        return cls(dbpool=dbpool)

    def process_item(self,item,spider):

        query=self.dbpool.runInteraction(self.do_insert, item)
        #执行，相当于loop启动，定义一个变量来代表
        query.addErrback(self.handler)
        #制定错误处理函数

    def do_insert(self,cursor,item):
        '''执行查询的函数'''
        a = {k: v[0] for k, v in item.items()}  # 构造字符串
        str1 = ",".join([i for i in a.keys()])
        str2 = "','".join([i for i in a.values()])
        sql = "INSERT INTO jianshu ({0}) VALUES ('{1}')".format(str1, str2)  # 构造sql
        cursor.execute(sql)

        #不能用try异步没有try
    def handler(self,failure):#定义错误处理
        print(failure)#打印错误

class JsonPipeline(object):
    def process_item(self,item,spider):
        a = {k: v[0] for k, v in item.items()}#格式化一下
        with open(r'jianshu.json','a',encoding='utf-8') as f:
            json.dump(a,f,ensure_ascii=False)
            f.write('\n')

import fake_useragent
class RandomUAMiddleware(object):
    def process_request(self,request,spider):
        ua=fake_useragent.UserAgent()
        request.headers.setdefault('User-Agent',ua.random)

from fake_useragent import UserAgent
ua = UserAgent()
ua.random#直接随机返回一个头
ua.ie#返回ie的头

import random
#from settings import IPpool
class IPproxyMiddleware(object):
    def process_request(self,request):
        '''引入一个ip池或请求一次(购买的)接口,随机选择一个ip就可以'''
        request.meta["proxy"] = random.choice(["https://222.223.115.30:51618"])

class cookieMiddleware(object):
    def process_request(self,request,spider):
        request.cookies ='通过接口或什么方式拿到你的cookie'

    def process_response(self, request, response, spider):
        '''
        对此次响应进行处理
        如果请求不成功怎么处理
        如果cookie失效怎么办
        '''
        pass

class seleniumMiddleware(object):
    def __init__(self):
        self.broswer = webdriver.Chrome(executable_path=r'D:\57635\chromedriver.exe')
        super().__init__()
    def process_request(self,request,spider):
        if spider.name=='jianshu':
            self.broswer.get(request.url)
            time.sleep(2)
            print('selenium访问{}'.format(request.url))

            return HtmlResponse(url=self.broswer.current_url,
                                body=self.broswer.page_source,
                                encoding='utf-8',
                                request=request)

from selenium import webdriver
from scrapy.xlib.pydispatch import dispatcher
from scrapy import signals

...
    def __init__(self):
        self.broswer = webdriver.Chrome(executable_path=r'D:\57635\chromedriver.exe')
        super().__init__()
        dispatcher.connect(self.spider_closed,signals.spider_closed)#信号量

    def spider_closed(self,spider):
        print("完成")
        self.broswer.quit()#退出

...

middlware:
import time
from scrapy.http import HtmlResponse
class seleniumMiddleware(object):

    def process_request(self,request,spider):
        if spider.name=='jianshu':
            spider.broswer.get(request.url)
            time.sleep(2)
            print('selenium访问{}'.format(request.url))

            return HtmlResponse(url=spider.broswer.current_url,
                                body=spider.broswer.page_source,
                                encoding='utf-8',
                                request=request)