2.3.4 提交新请求

from bricks import Request, const
from bricks.core import events
from bricks.plugins import scripts
from bricks.spider import air
from loguru import logger
 
 
class MySpider(air.Spider):
 
    def make_seeds(self, context: air.Context, **kwargs):
        return {"page": 1}
 
    def make_request(self, context: air.Context) -> Request:
        # 之前定义的种子会被投放至任务队列, 之后会被取出来, 迁入至 context 对象内
        seeds = context.seeds
        return Request(
            url="https://fx1.service.kugou.com/mfanxing-home/h5/cdn/room/index/list_v2",
            params={
                "page": seeds["page"],
                "cid": 6000
            },
            headers={
                "User-Agent": "Mozilla/5.0 (Linux; Android 10; Redmi K30 Pro) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Mobile Safari/537.36",
                "Content-Type": "application/json;charset=UTF-8",
            },
        )
 
    def parse(self, context: air.Context):
        response = context.response
        return response.extract(
            engine="json",
            rules={
                "data.list": {
                    "userId": "userId",
                    "roomId": "roomId",
                    "score": "score",
                    "startTime": "startTime",
                    "kugouId": "kugouId",
                    "status": "status",
                }
            }
        )
 
    def item_pipeline(self, context: air.Context):
        items = context.items
        # 写自己的存储逻辑
        logger.debug(f'存储: {items}')
        # 确认种子爬取完毕后删除, 不删除的话后面又会爬取
        context.success()
 
    def install(self):
        super().install()
        self.use(
            const.AFTER_REQUEST,
            events.Task(
                func=scripts.is_success,
                kwargs={
                    "match": [
                        "context.response.get('code') == 0"
                    ]
                }
            )
        )
 
    @staticmethod
    @events.on(const.BEFORE_PIPELINE)
    def turn_page(context: air.Context):
        # 判断是否存在下一页
        has_next = context.response.get('data.hasNextPage')
        if has_next == 1:
            # 提交翻页的种子
            context.submit({**context.seeds, "page": context.seeds["page"] + 1})
 
 
if __name__ == '__main__':
    spider = MySpider()
    spider.run()