ElasticSearch 知识点整理

灰太狼 2022-12-30 13:56 294阅读 0赞

**目录**

一、前文

二、概念

2.1 集群

2.2 读写请求

2.3 优化

2.3.1 部署调优

2.3.2 配置调优

2.3.3 DSL调优

2.4 其他

三、实战

3.1 Mysql - Es 映射

3.2 Hive - Es映射

--------------------

# 一、前文 #

目前公司有3套ES集群环境：直播业务集群(6节点)，论坛业务集群(3节点)，日志集群(3节点)，应用场景分别为线上直播及订单商品等业务查询、大数据报表数据查询、日志监控等。其中使用Filebeat，Logstash，SparkStreaming 等接入数据，Kibana 负责监控。

# 二、概念 #

## 2.1 集群 ##

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MjY4NzA3NA_size_16_color_FFFFFF_t_70][]

## 2.2 读写请求 ##

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MjY4NzA3NA_size_16_color_FFFFFF_t_70 1][]

**注**：该图为其他博主所画，如有侵权请联系删除。

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MjY4NzA3NA_size_16_color_FFFFFF_t_70 2][]

## 2.3 优化 ##

### 2.3.1 部署调优 ###

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MjY4NzA3NA_size_16_color_FFFFFF_t_70 3][]

### 2.3.2 配置调优 ###

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MjY4NzA3NA_size_16_color_FFFFFF_t_70 4][]

### 2.3.3 DSL调优 ###

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MjY4NzA3NA_size_16_color_FFFFFF_t_70 5][]

## 2.4 其他 ##

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MjY4NzA3NA_size_16_color_FFFFFF_t_70 6][]

# 三、实战 #

## 3.1 Mysql - Es 映射 ##

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MjY4NzA3NA_size_16_color_FFFFFF_t_70 7][]

优化索引：

PUT xxxxxx/
    {
      "settings": {
        "index.search.slowlog.threshold.query.warn": "4s",
        "index.search.slowlog.threshold.query.info": "2s",
        "index.search.slowlog.threshold.query.debug": "1s",
        "index.search.slowlog.threshold.query.trace": "100ms",
        "index.search.slowlog.threshold.fetch.warn": "2s",
        "index.search.slowlog.threshold.fetch.info": "1s",
        "index.search.slowlog.threshold.fetch.debug": "-1",
        "index.search.slowlog.threshold.fetch.trace": "-1",
        "index.search.slowlog.level": "info",
        "refresh_interval": "10s",
        "index.merge.policy.floor_segment": "50m",
        "index.merge.policy.max_merged_segment": "2gb",
        "number_of_shards": 5,
        "number_of_replicas": 1
      },
      "mappings": {
        "dynamic_date_formats": [
          "yyyy-MM-dd HH:mm:ss"
        ],
        "properties": {
          "Id": {
            "type": "keyword",
            "doc_values": false
          },
          "userId": {
            "type": "keyword",
            "doc_values": false
          },
          "Name": {
            "type": "text",
            "analyzer": "ik_max_word",
            "search_analyzer": "ik_smart"
          },
          "mininsnsId": {
            "type": "keyword",
            "doc_values": false
          },
          "TplName": {
            "type": "keyword",
            "index": false,
            "doc_values": false
          },
          "Descript": {
            "type": "text",
            "index": false,
            "doc_values": false
          },
          "state": {
            "type": "keyword",
            "doc_values": false
          },
          "MyQRCode": {
            "type": "keyword",
            "index": false,
            "doc_values": false
          },
          "IsOpenNew": {
            "type": "keyword",
            "index": false,
            "doc_values": false
          },
          "Levels": {
            "type": "keyword",
            "doc_values": false
          },
          "area": {
            "type": "keyword",
            "index": false,
            "doc_values": false
          },
          "isOpenplat": {
            "type": "keyword",
            "doc_values": false
          },
          "province": {
            "type": "keyword",
            "index": false,
            "doc_values": false
          },
          "city": {
            "type": "keyword",
            "index": false,
            "doc_values": false
          },
          "CustomerType": {
            "type": "keyword",
            "doc_values": false
          },
          "addtime": {
            "type": "date",
            "format": "yyyy-MM-dd HH:mm:ss"
          },
          "Setting": {
            "type": "keyword",
            "index": false,
            "doc_values": false
          },
          "applyId": {
            "type": "keyword",
            "doc_values": false
          },
          "isFlowPay": {
            "type": "keyword",
            "doc_values": false
          }
        }
      }
    }

## 3.2 Hive - Es映射 ##

drop table ads.es_cdnlog;
    create table ads.es_cdnlog(
    createdate string comment '流量日期',
    zbid bigint comment '直播id',
    stream string comment '流量id',
    livetraffic bigint comment '直播流量',
    backtraffic bigint comment '回放流量'
    )
    stored	by 'org.elasticsearch.hadoop.hive.EsStorageHandler'
    tblproperties(
    'es.nodes' = 'es_01:9200,es_02:9200',
    'es.resource' = 'cdn_traffic_day',
    'es.mapping.date.rich'='false',
    'es.date.format'='yyyy-MM-dd',
    'es.net.http.auth.user'='elastic','es.net.http.auth.pass'='123456',
    'es.mapping.names' = '
    createdate:createDate,
    zbid:zbId,
    stream:streamId,
    livetraffic:liveTrafficSum,
    backtraffic:playbackTrafficSum');

注：在Hive 放入官网找到的 elasticsearch-for-apache-hadoop.jar 绑定即可

<property>
        <name>hive.reloadable.aux.jars.path</name>
        <value>/home/hadoopjar/</value>
    </property>

## 3.3 重建索引 ##

POST _reindex                   
    {
      "source": {
        "index": "order_transaction_v1"
      },
      "dest": {
        "index": "order_transaction_v2"
      }
    }
    
    POST /_aliases
    {
      "actions": [
        {
          "remove": {
            "index": "order_transaction_v1",
            "alias": "order_transaction_record"
          }
        },
        {
          "add": {
            "index": "order_transaction_v2",
            "alias": "order_transaction_record"
          }
        }
      ]
    }

## 3.4 Kibana 监控等 ##

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MjY4NzA3NA_size_16_color_FFFFFF_t_70 8][]

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MjY4NzA3NA_size_16_color_FFFFFF_t_70]: /images/20221120/4dfed59f1b2648c6b378ce628ac5d5b1.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MjY4NzA3NA_size_16_color_FFFFFF_t_70 1]: /images/20221120/c3031e394ada458abb84cf81f925cc14.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MjY4NzA3NA_size_16_color_FFFFFF_t_70 2]: /images/20221120/84d3d3f1cead4870b9088dcc3e213790.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MjY4NzA3NA_size_16_color_FFFFFF_t_70 3]: /images/20221120/7b473b6374cf42fa8833b779eae35fd6.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MjY4NzA3NA_size_16_color_FFFFFF_t_70 4]: /images/20221120/98946136669246c9966c4ae46bb21a95.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MjY4NzA3NA_size_16_color_FFFFFF_t_70 5]: https://img-blog.csdnimg.cn/20201226110730683.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MjY4NzA3NA==,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MjY4NzA3NA_size_16_color_FFFFFF_t_70 6]: https://img-blog.csdnimg.cn/20201226110833501.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MjY4NzA3NA==,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MjY4NzA3NA_size_16_color_FFFFFF_t_70 7]: https://img-blog.csdnimg.cn/20201226110920807.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MjY4NzA3NA==,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MjY4NzA3NA_size_16_color_FFFFFF_t_70 8]: https://img-blog.csdnimg.cn/20201226112235526.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MjY4NzA3NA==,size_16,color_FFFFFF,t_70