scrapy 管道 pipe 异步操作

maiyantao

1. from twisted.internet.threads import deferToThread
   
2. 
   
3. 
   
4. class RedisPipeline:
   
5. 
   
6.     def __init__(self):
   
7.         pass
   
8. 
   
9.     def process_item(self, item, spider):
   
10.         return deferToThread(self._process_item, item, spider)
    
11. 
    
12.     def _process_item(self, item, spider):
    
13.         return item

复制代码