SpringCloudAlibaba--Sentinel熔断、熔断框架比较

迷南。 2021-08-27 16:36 497阅读 0赞

### Sentinel服务熔断 ###

*  sentinel整合ribbon+openFeign+fallback
 *  Ribbon：
    
     *  提供者9003/9004
     *  消费者84
 *  Feign：

**新建cloudalibaba-provider-payment9003/9004**

依赖：

<?xml version="1.0" encoding="UTF-8"?>
    <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
        <parent>
            <artifactId>cloud2020</artifactId>
            <groupId>pers.zhang.springcloud</groupId>
            <version>1.0-SNAPSHOT</version>
        </parent>
        <modelVersion>4.0.0</modelVersion>
    
        <artifactId>cloudalibaba-provider-payment9003</artifactId>
    
        <dependencies>
            
            <dependency>
                <groupId>com.alibaba.cloud</groupId>
                <artifactId>spring-cloud-starter-alibaba-nacos-discovery</artifactId>
            </dependency>
            <dependency>
                <groupId>pers.zhang.springcloud</groupId>
                <artifactId>cloud-api-commons</artifactId>
                <version>${project.version}</version>
            </dependency>
            
            <dependency>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-starter-web</artifactId>
            </dependency>
            <dependency>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-starter-actuator</artifactId>
            </dependency>
            
            <dependency>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-devtools</artifactId>
                <scope>runtime</scope>
                <optional>true</optional>
            </dependency>
            <dependency>
                <groupId>org.projectlombok</groupId>
                <artifactId>lombok</artifactId>
                <optional>true</optional>
            </dependency>
            <dependency>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-starter-test</artifactId>
                <scope>test</scope>
            </dependency>
        </dependencies>
    
    </project>

配置：application.yml

server:
      port: 9003
    spring:
      application:
        name: nacos-payment-provider
      cloud:
        nacos:
          discovery:
            server-addr: localhost:8848
    
    management:
      endpoints:
        web:
          exposure:
            include: '*'

Controller：

@RestController
    public class PaymentController { 
    
        @Value("${server.port}")
        String serverPort;
    
        public static HashMap<Long, Payment> hashMap = new HashMap<>();
    
        static { 
            hashMap.put(1L, new Payment(1L, "1111111111111"));
            hashMap.put(2L, new Payment(2L, "2222222222222"));
            hashMap.put(3L, new Payment(3L, "3333333333333"));
        }
    
        @GetMapping(value = "/paymentSQL/{id}")
        public CommonResult<Payment> paymentSQL(@PathVariable("id") Long id) { 
            Payment payment = hashMap.get(id);
            CommonResult<Payment> result = new CommonResult(200, "from mysql, serverPort: " + serverPort, payment);
            return result;
        }
    }

启动类：

@SpringBootApplication
    @EnableDiscoveryClient
    public class Payment9003 { 
        public static void main(String[] args) { 
            SpringApplication.run(Payment9003.class, args);
        }
    }
    ```**新建cloudalibaba-consumer-nacos-order84**
    依赖：
    ```xml
    <?xml version="1.0" encoding="UTF-8"?>
    <project xmlns="http://maven.apache.org/POM/4.0.0"
             xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
             xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
        <parent>
            <artifactId>cloud2020</artifactId>
            <groupId>pers.zhang.springcloud</groupId>
            <version>1.0-SNAPSHOT</version>
        </parent>
        <modelVersion>4.0.0</modelVersion>
    
        <artifactId>cloudalibaba-consumer-nacos-order84</artifactId>
    
        <dependencies>
            
            <dependency>
                <groupId>org.springframework.cloud</groupId>
                <artifactId>spring-cloud-starter-openfeign</artifactId>
            </dependency>
            
            <dependency>
                <groupId>com.alibaba.cloud</groupId>
                <artifactId>spring-cloud-starter-alibaba-nacos-discovery</artifactId>
            </dependency>
            
            <dependency>
                <groupId>com.alibaba.cloud</groupId>
                <artifactId>spring-cloud-starter-alibaba-sentinel</artifactId>
            </dependency>
            
            <dependency>
                <groupId>pers.zhang.springcloud</groupId>
                <artifactId>cloud-api-commons</artifactId>
                <version>${ project.version}</version>
            </dependency>
            
            <dependency>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-starter-web</artifactId>
            </dependency>
            <dependency>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-starter-actuator</artifactId>
            </dependency>
            
            <dependency>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-devtools</artifactId>
                <scope>runtime</scope>
                <optional>true</optional>
            </dependency>
            <dependency>
                <groupId>org.projectlombok</groupId>
                <artifactId>lombok</artifactId>
                <optional>true</optional>
            </dependency>
            <dependency>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-starter-test</artifactId>
                <scope>test</scope>
            </dependency>
        </dependencies>
    
    </project>

配置：application.yml

server:
      port: 84
    
    spring:
      application:
        name: nacos-order-consumer
      cloud:
        nacos:
          discovery:
            server-addr: localhost:8848
        sentinel:
          transport:
            dashboard: localhost:8080
            port: 8719
    
    service-url:
      nacos-user-service: http://nacos-payment-provider

@Configuration
    public class ApplicationContextConfig { 
        @Bean
        @LoadBalanced
        public RestTemplate getTemplate() { 
            return new RestTemplate();
        }
    }

Controller:

@RestController
    @Slf4j
    public class CircleBreakerController { 
    
        public static final String SERVICE_URL = "http://nacos-payment-provider";
    
        @Resource
        RestTemplate restTemplate;
    
        @RequestMapping("/consumer/fallback/{id}")
        @SentinelResource(value = "fallback")
        //@SentinelResource(value = "fallback", fallback = "handlerFallback")//fallback只负责业务异常
        //@SentinelResource(value = "fallback", blockHandler = "blockHandler")//blockHandler只负责sentinel控制台配置违规
        //@SentinelResource(value = "fallback", fallback = "handlerFallback", blockHandler = "blockHandler")
        public CommonResult<Payment> fallback(@PathVariable Long id) { 
            CommonResult<Payment> result = restTemplate.getForObject(SERVICE_URL + "/paymentSQL/" + id, CommonResult.class);
            if (id == 4) { 
                throw new IllegalArgumentException("IllegalArgumentException 非法参数异常...");
            } else if (result.getData() == null) { 
                throw new NullPointerException("NullPointerException 空指针异常...");
            }
            return result;
        }
    
        //本例是fallback
        public CommonResult handlerFallback(@PathVariable  Long id,Throwable e) { 
            Payment payment = new Payment(id,"null");
            return new CommonResult<>(444,"兜底异常handlerFallback,exception内容 "+e.getMessage(),payment);
        }
        //本例是blockHandler
        public CommonResult blockHandler(@PathVariable  Long id, BlockException blockException) { 
            Payment payment = new Payment(id,"null");
            return new CommonResult<>(445,"blockHandler-sentinel限流,无此流水: blockException "+blockException.getMessage(),payment);
        }

启动类：

@SpringBootApplication
    @EnableDiscoveryClient
    public class OrderNacosMain84 { 
        public static void main(String[] args) { 
            SpringApplication.run(OrderNacosMain84.class, args);
        }
    }

**测试：**  
启动nacos、Sentinel、9003、9004以及84微服务：  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70]

访问：`localhost:84/consumer/fallback/id`  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 1]  
id为4，返回非法参数  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 2]  
id为其它，返回空指针  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 3]

直接返回错误不够友好，所以我们添加fallback方法。  
修改@SentinelResource，并添加fallback方法：  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 4]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 5]  
重启84，访问：`localhost:84/consumer/fallback/id`  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 6]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 7]

修改@SentinelResource，只配置blockHandler：  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 8]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 9]

配置降级规则：  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 10]  
重启84，访问：`localhost:84/consumer/fallback/id`，第一次结果如下：  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 11]  
访问：`localhost:84/consumer/fallback/id`，（2秒内）第三次结果如下：  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 12]  
触发blockHandler。

**同时配置fallback和blockHandler**  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 13]

流控配置：  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 14]

重启84，快速多次访问(1秒内超过1次)：`localhost:84/consumer/fallback/1`  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 15]  
`结论：程序本身无异常，如果触发blockHandler则执行blockHandler。`

单次访问：`localhost:84/consumer/fallback/4`：  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 16]  
快速多次访问(1秒内超过1次)：`localhost:84/consumer/fallback/4`  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 17]  
`结论：在未触发blockHandler的情况下，程序出现异常则执行fallback；如果触发blockHandler，优先执行BlockHandler；`

**异常忽略：**  
忽略非法参数异常,假如报该异常，不再有fallback方法兜底，失去降级效果  
![在这里插入图片描述][20200323003655195.png]  
重启84，访问：`localhost:84/consumer/fallback/4`  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 18]

#### 整合Feign ####

增加依赖：

<dependency>
                <groupId>org.springframework.cloud</groupId>
                <artifactId>spring-cloud-starter-openfeign</artifactId>
            </dependency>

添加配置：

#激活sentinel对feign的支持
    feign:
      sentinel:
        enabled: true

启动类：

@SpringBootApplication
    @EnableDiscoveryClient
    @EnableFeignClients//开启Feign
    public class OrderNacosMain84 { 
        public static void main(String[] args) { 
            SpringApplication.run(OrderNacosMain84.class, args);
        }
    }

新增接口：

@FeignClient(value = "nacos-payment-provider", fallback = PaymentFallbackService.class)
    public interface PaymentService { 
    
        @GetMapping(value = "/paymentSQL/{id}")
        public CommonResult<Payment> paymentSQL(@PathVariable("id") Long id);
    }

兜底类：

@Component
    public class PaymentFallbackService implements PaymentService { 
        @Override
        public CommonResult<Payment> paymentSQL(Long id) { 
            return new CommonResult<>(444, "服务降级返回，-----PaymentFallbackService",new Payment(id, "errorSerial"));
        }
    }

Controller:

//==================OpenFeign
        @Resource
        private PaymentService paymentService;
    
        @GetMapping(value = "/consumer/paymentSQL/{id}")
        public CommonResult<Payment> paymentSQL(@PathVariable("id") Long id)
        { 
            return paymentService.paymentSQL(id);
        }

测试：启动nacos，启动sentinel，启动9003、9004和84微服务：

访问：`localhost:84/consumer/paymentSQL/1`  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 19]

测试84调用9003/9004，此时故意关闭9003/9004微服务提供者，看84消费侧自动降级，不会被耗死：  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 20]

### 熔断框架比较 ###

<table> 
 <thead> 
  <tr> 
   <th align="left"></th> 
   <th align="left">Sentinel</th> 
   <th align="left">Hystrix</th> 
   <th align="left">resilience4j</th> 
  </tr> 
 </thead> 
 <tbody> 
  <tr> 
   <td align="left">隔离策略</td> 
   <td align="left">信号量隔离（并发线程数限流）</td> 
   <td align="left">线程池隔离/信号量隔离</td> 
   <td align="left">信号量隔离</td> 
  </tr> 
  <tr> 
   <td align="left">熔断降级策略</td> 
   <td align="left">基于时间响应、异常比率、异常数</td> 
   <td align="left">基于异常比率</td> 
   <td align="left">基于异常比率、响应时间</td> 
  </tr> 
  <tr> 
   <td align="left">实时统计实现</td> 
   <td align="left">滑动窗口（LeapArray）</td> 
   <td align="left">滑动窗口（基于RxJava）</td> 
   <td align="left">Ring Bit Buffer</td> 
  </tr> 
  <tr> 
   <td align="left">动态规则配置</td> 
   <td align="left">支持多种数据源</td> 
   <td align="left">支持多种数据源</td> 
   <td align="left">有限支持</td> 
  </tr> 
  <tr> 
   <td align="left">扩展性</td> 
   <td align="left">多个扩展点</td> 
   <td align="left">插件形式</td> 
   <td align="left">接口形式</td> 
  </tr> 
  <tr> 
   <td align="left">基于注解的支持</td> 
   <td align="left">支持</td> 
   <td align="left">支持</td> 
   <td align="left">支持</td> 
  </tr> 
  <tr> 
   <td align="left">限流</td> 
   <td align="left">基于QPS、支持基于调用关系的限流</td> 
   <td align="left">有限的支持</td> 
   <td align="left">Rate Limiter</td> 
  </tr> 
 </tbody> 
</table>

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70]: /images/20210813/85feb79dba994b3299401aa98dae584b.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 1]: /images/20210813/a081211550d94230b90c7d61de31d2ba.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 2]: /images/20210813/3129cf64066c42b3bf14ffafbd22c967.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 3]: /images/20210813/e53ee5a2702d48f8aa47eafc5116132e.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 4]: /images/20210813/c490d5c2086540a7bf64751ebd329820.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 5]: /images/20210813/ded274db45ce4138b191868c5d23e199.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 6]: /images/20210813/2572fc08aded4504afad2ac312494a17.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 7]: /images/20210813/9d4df77a19184e83828eee4bb041e309.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 8]: /images/20210813/5059e269e61d4644b4802e6218af2844.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 9]: /images/20210813/944757ca1a41493a91f37c8cf04fb930.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 10]: /images/20210813/db1d72a9be454a718572777158bea93a.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 11]: /images/20210813/de012b21d065499ca525fa9df412bb0f.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 12]: /images/20210813/e566782ef4444326a830731cdf96d721.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 13]: /images/20210813/66e54030dfb248bea49aaab7c03d1e31.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 14]: /images/20210813/032b073ecc304223b5011a895501608a.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 15]: /images/20210813/df18ea72a96c4d6e98f8b57760a84256.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 16]: /images/20210813/ab5ddc564b264fd7ac2c185a270ea4a8.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 17]: /images/20210813/6d39e3984a33459eaeb831079afe02c6.png
[20200323003655195.png]: /images/20210813/a0ad7273540f45409dc4ec11e2fbeba4.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 18]: /images/20210813/fe34d6f707de4eb7af9458c34531a885.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 19]: /images/20210813/bda6be8a2a154e2aad23de8a970d9697.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2NvbGRfX19wbGF5_size_16_color_FFFFFF_t_70 20]: /images/20210813/e51c1d8bf1c941c4b3782fd1a1056a5e.png