elasticsearch使用scroll游标查询全量数据（极简）

阳光穿透心脏的1/2处 2022-01-06 11:03 1071阅读 0赞

使用elasticsearchTemplate实现,各种筛选条件可以自己在NativeSearchQueryBuilder 中添加

import com.vivo.admin_log_web.entity.AdminLog;
    import org.elasticsearch.index.query.QueryBuilders;
    import org.springframework.beans.factory.annotation.Autowired;
    import org.springframework.data.domain.PageRequest;
    import org.springframework.data.elasticsearch.core.ElasticsearchTemplate;
    import org.springframework.data.elasticsearch.core.ScrolledPage;
    import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
    import org.springframework.stereotype.Service;
    
    import java.util.ArrayList;
    import java.util.HashMap;
    import java.util.List;
    
    /** * @author *** */
    @Service
    public class DimissionLogExportServiceImpl { 
        /** * scroll游标超时时间，单位ms */
        private final long SCROLL_TIMEOUT = 30000;
        /** * scroll游标分页每次查询条数 */
        private int SCROLL_PageSize = 5000;
    
        @Autowired
        private ElasticsearchTemplate esTemplate;
    
        public List<AdminLog> getDimissionLogList(HashMap<String, Object> paramsMap) { 
            ArrayList<AdminLog> resultAdminLogs = new ArrayList<>();
            NativeSearchQueryBuilder nsQueryBuilder = new NativeSearchQueryBuilder();
            nsQueryBuilder.withQuery(QueryBuilders.queryStringQuery("*:*"));
            nsQueryBuilder.withPageable(PageRequest.of(0,SCROLL_PageSize));
            ScrolledPage<AdminLog> scrollAdminLog = (ScrolledPage<AdminLog>) esTemplate.startScroll(SCROLL_TIMEOUT, nsQueryBuilder.build(), AdminLog.class);
            while (scrollAdminLog.hasContent()){ 
                for (AdminLog ad:scrollAdminLog) { 
                    resultAdminLogs.add(ad);
                }
                //取下一页，scrollId在es服务器上可能会发生变化，需要用最新的;发起continueScroll请求会重新刷新快照保留时间
                scrollAdminLog = (ScrolledPage<AdminLog>) esTemplate.continueScroll(scrollAdminLog.getScrollId(), SCROLL_TIMEOUT, AdminLog.class);
    
            }
            //及时清除es快照，释放资源
            esTemplate.clearScroll(scrollAdminLog.getScrollId());
            return resultAdminLogs;
        }
    }

pom依赖

<dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-data-elasticsearch</artifactId>
    </dependency>