java使用多线程进行分页数据采集

た入场券 2022-10-01 02:47 213阅读 0赞

public  void executeList(Date startDate, Date endDate,String subgamekind) {
       System.out.println("定时任务：抓取任务--->开始------->"+ sd1.format(new Date()));
       long start1 = System.currentTimeMillis();
       int totalPage = 0;
    
       try {
          String start = sd1.format(startDate);
          String end = sd1.format(endDate);
          BBINBetResultApi result = GameUtil.getBBINBetRecordResult(start, end, "5", "1","",subgamekind);
          if (result != null) {
             if(result.getResult().equals("true")){
                if(result.getPagination()!=null){
                   totalPage = Integer.valueOf(result.getPagination().getTotalPage());
                }else {
                   totalPage = 0;
                }
             }
          }
          if(totalPage!=0) {
             // 创建一个线程池
             ThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(totalPage);
             CountDownLatch countDownLatch = new CountDownLatch(totalPage);
             for (int i = 0; i < totalPage; i++) {
                if(i==0){
                   BBINBetsRunnable bbinBetsRunnable = new BBINBetsRunnable(start,end,"5",String.valueOf(i + 1),"",countDownLatch,bbinBetsLstMapper,bbinBetsSlotMapper,subgamekind,"single",result!=null?result.getData():null);
                   executor.execute(bbinBetsRunnable);
                   continue;
                }
                Thread.sleep(2000);
                BBINBetsRunnable bbinBetsRunnable = new BBINBetsRunnable(start,end,"5",String.valueOf(i + 1),"",countDownLatch,bbinBetsLstMapper,bbinBetsSlotMapper,subgamekind,"multi",null);
                executor.execute(bbinBetsRunnable);
    
             }
             countDownLatch.await();
             executor.shutdown();
          }
       } catch (Exception e) {
          e.printStackTrace();
       }
       long end1 = System.currentTimeMillis();
       logger.info("定时任务：抓取任务--->结束" + sd.format(new Date())+" 定时插入耗时："+(end1-start1));
    }

package com.ig.bbin.service.impl;
    
    import com.ig.bbin.dao.BBINBetsLstMapper;
    import com.ig.bbin.dao.BBINBetsSlotMapper;
    import com.ig.biservice.config.GetUserCodeUtil;
    import com.ig.biservice.config.SetCustMap;
    import com.ig.common.util.UtilTool;
    import com.ig.gapi.result.bbin.BBINBetResultApi;
    import com.ig.gapi.result.bbin.BBINbet;
    import com.ig.oths.util.GameUtil;
    import com.ig.sid.syssetting.domain.SidCustMap;
    import org.apache.log4j.Logger;
    
    import java.text.SimpleDateFormat;
    import java.util.ArrayList;
    import java.util.List;
    import java.util.concurrent.CountDownLatch;
    
    /** * */ public class BBINBetsRunnable implements Runnable {
        private static final Logger logger = Logger.getLogger(BBINBetsRunnable.class);
    
        private String startDate ;
    
        private String endDate ;
    
        private String pageNum;
    
        private String gameKind;
    
        private String gameType;
    
        private String subgamekind;
    
        private BBINBetsLstMapper bbinBetLstMapper;
    
        private BBINBetsSlotMapper bbinBetSlotsMapper;
    
        private CountDownLatch countDownLatch ;
    
        private String type;
    
        private BBINbet[] bbinBet_Data;
    
        public BBINBetsRunnable(String startDate, String endDate,String gameKind, String pageNum,String gameType, CountDownLatch countDownLatch, BBINBetsLstMapper bbinBetLstMapper,BBINBetsSlotMapper bbinBetSlotsMapper,String subgamekind,String type,BBINbet[] bbinBet_Data){
    
            this.startDate  = startDate ;
    
            this.endDate = endDate ;
    
            this.gameKind = gameKind;
    
            this.pageNum = pageNum;
    
            this.gameType = gameType;
    
            this.bbinBetLstMapper = bbinBetLstMapper ;
    
            this.bbinBetSlotsMapper = bbinBetSlotsMapper;
    
            this.countDownLatch = countDownLatch ;
    
            this.subgamekind= subgamekind;
    
            this.type = type;
    
            this.bbinBet_Data=bbinBet_Data;
    
        }
    
        @Override
        public void run() {
            try {
                BBINbet[] bbinBetData = null;
                System.out.println("--------------------- BetRecord   注单抓取  --------------------------");
                if(type.equals("multi")){
                    BBINBetResultApi bbinBetResultApi = GameUtil.getBBINBetRecordResult(startDate, endDate,gameKind, pageNum,gameType,subgamekind);
                    if(null!=bbinBetResultApi && bbinBetResultApi.getResult().equals("true")){
                        if(null!=bbinBetResultApi.getData()){
                            bbinBetData = bbinBetResultApi.getData();
                        }
                    }
                }else{
                    bbinBetData = bbinBet_Data;
                }
    
    
                BBINbet bbiNbet = null;
                List<BBINbet> bbiNbetList = new ArrayList<BBINbet>();
                if(bbinBetData!=null && bbinBetData.length>0) {
                    if (GetUserCodeUtil.custMap.isEmpty()) {
                        SetCustMap setCustMap = new SetCustMap();
                        setCustMap.addCustMap();
                    }
                }
                if(bbinBetData!=null) {
                    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                    //2.入库
                    for(int i=0;i<bbinBetData.length;i++){
                        bbiNbet = bbinBetData[i];
                        String userName = bbiNbet.getUserName();
                        bbiNbet.setGameKind(gameKind);
                        bbiNbet.setWagersDate(bbiNbet.getWagersDate() == null ? null : UtilTool.headDate(sdf.format(bbiNbet.getWagersDate()), 12));
    
                        long hallId = 0l;
                        String gamePwd = "0";
                        SidCustMap custmapobj = GetUserCodeUtil.custCodeMap.get(userName);
                        if(null==custmapobj){
                            String newUserName = userName.toUpperCase();
                            SidCustMap custobj = GetUserCodeUtil.custCodeMap.get(newUserName);
                            if(null!=custobj){
                                gamePwd = custobj.getGamePwd();
                                hallId = custobj.getHallId();
                            }
                        }else {
                            gamePwd = custmapobj.getGamePwd();
                            hallId = custmapobj.getHallId();
                        }
    
                        bbiNbet.setHallId(hallId);//GetUserCodeUtil.custCodeMap.get(userName) == null ? 0l : GetUserCodeUtil.custCodeMap.get(userName).getHallId()
                        bbiNbet.setUserId(Long.valueOf(gamePwd));//GetUserCodeUtil.custCodeMap.get(userName) == null ? 0l : Long.valueOf(GetUserCodeUtil.custCodeMap.get(userName).getGamePwd())
                        if(bbiNbet.getHallId()==0l){
                            continue;
                        }
                        bbiNbetList.add(bbiNbet);
                    }
    
                    if (bbiNbetList.size() > 0) {
                     if(gameKind.equals("1") || gameKind.equals("3") || gameKind.equals("12") || gameKind.equals("30")){
                            bbinBetLstMapper.replaceBBinBet(bbiNbetList);
                        }else {
                            bbinBetSlotsMapper.replaceBBinBet(bbiNbetList);
                        }
    
                    }
                }
            } catch (Exception e) {
                logger.error("插入数据异常，异常消息："+e);
                e.printStackTrace();
            } finally {
                countDownLatch.countDown();
            }
        }
    }