9.Flume Sinks

小灰灰 2022-06-14 07:21 160阅读 0赞

1.Hdfs Sink

a1.channels = c1
    a1.sinks = k1
    a1.sinks.k1.type = hdfs
    a1.sinks.k1.channel = c1
    a1.sinks.k1.hdfs.path = /flume/events/%y-%m-%d/%H%M/%S
    a1.sinks.k1.hdfs.filePrefix = events-
    a1.sinks.k1.hdfs.round = true
    a1.sinks.k1.hdfs.roundValue = 10
    a1.sinks.k1.hdfs.roundUnit = minute

2.Hive Sink

a1.channels = c1
    a1.channels.c1.type = memory
    a1.sinks = k1
    a1.sinks.k1.type = hive
    a1.sinks.k1.channel = c1
    a1.sinks.k1.hive.metastore = thrift://127.0.0.1:9083
    a1.sinks.k1.hive.database = logsdb
    a1.sinks.k1.hive.table = weblogs
    a1.sinks.k1.hive.partition = asia,%{country},%y-%m-%d-%H-%M
    a1.sinks.k1.useLocalTimeStamp = false
    a1.sinks.k1.round = true
    a1.sinks.k1.roundValue = 10
    a1.sinks.k1.roundUnit = minute
    a1.sinks.k1.serializer = DELIMITED
    a1.sinks.k1.serializer.delimiter = "\t"
    a1.sinks.k1.serializer.serdeSeparator = '\t'
    a1.sinks.k1.serializer.fieldnames =id,,msg

3.Logger Sink

a1.channels = c1
    a1.sinks = k1
    a1.sinks.k1.type = logger
    a1.sinks.k1.channel = c1

4.Avro Sink

a1.channels = c1
    a1.sinks = k1
    a1.sinks.k1.type = avro
    a1.sinks.k1.channel = c1
    a1.sinks.k1.hostname = 10.10.10.10
    a1.sinks.k1.port = 4545

5.Thrift Sink

a1.channels = c1
    a1.sinks = k1
    a1.sinks.k1.type = thrift
    a1.sinks.k1.channel = c1
    a1.sinks.k1.hostname = 10.10.10.10
    a1.sinks.k1.port = 4545

6.IRC Sink

a1.channels = c1
    a1.sinks = k1
    a1.sinks.k1.type = irc
    a1.sinks.k1.channel = c1
    a1.sinks.k1.hostname = irc.yourdomain.com
    a1.sinks.k1.nick = flume
    a1.sinks.k1.chan = #flume

7.File Roll Sink

a1.channels = c1
    a1.sinks = k1
    a1.sinks.k1.type = file_roll
    a1.sinks.k1.channel = c1
    a1.sinks.k1.sink.directory = /var/log/flume

8.NULL Sink

丢弃它从通道接收的所有事件。

a1.channels = c1
    a1.sinks = k1
    a1.sinks.k1.type = null
    a1.sinks.k1.channel = c1

9.Hbase Sink

a1.channels = c1
    a1.sinks = k1
    a1.sinks.k1.type = hbase
    a1.sinks.k1.table = foo_table
    a1.sinks.k1.columnFamily = bar_cf
    a1.sinks.k1.serializer = org.apache.flume.sink.hbase.RegexHbaseEventSerializer
    a1.sinks.k1.channel = c1

10.AsyncHbase Sink

a1.channels = c1
    a1.sinks = k1
    a1.sinks.k1.type = asynchbase
    a1.sinks.k1.table = foo_table
    a1.sinks.k1.columnFamily = bar_cf
    a1.sinks.k1.serializer = org.apache.flume.sink.hbase.SimpleAsyncHbaseEventSerializer
    a1.sinks.k1.channel = c1

11.MorphlineSolrSink

a1.channels = c1
    a1.sinks = k1
    a1.sinks.k1.type = org.apache.flume.sink.solr.morphline.MorphlineSolrSink
    a1.sinks.k1.channel = c1
    a1.sinks.k1.morphlineFile = /etc/flume-ng/conf/morphline.conf
    # a1.sinks.k1.morphlineId = morphline1
    # a1.sinks.k1.batchSize = 1000
    # a1.sinks.k1.batchDurationMillis = 1000

11.ElasticSearchSink

a1.channels = c1
    a1.sinks = k1
    a1.sinks.k1.type = elasticsearch
    a1.sinks.k1.hostNames = 127.0.0.1:9200,127.0.0.2:9300
    a1.sinks.k1.indexName = foo_index
    a1.sinks.k1.indexType = bar_type
    a1.sinks.k1.clusterName = foobar_cluster
    a1.sinks.k1.batchSize = 500
    a1.sinks.k1.ttl = 5d
    a1.sinks.k1.serializer = org.apache.flume.sink.elasticsearch.ElasticSearchDynamicSerializer
    a1.sinks.k1.channel = c1

12.Kite Dataset Sink

13.Kafka Sink

a1.sinks.k1.channel = c1
    a1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink
    a1.sinks.k1.kafka.topic = mytopic
    a1.sinks.k1.kafka.bootstrap.servers = localhost:9092
    a1.sinks.k1.kafka.flumeBatchSize = 20
    a1.sinks.k1.kafka.producer.acks = 1
    a1.sinks.k1.kafka.producer.linger.ms = 1
    a1.sinks.ki.kafka.producer.compression.type = snappy

14.Custom Sink

a1.channels = c1
    a1.sinks = k1
    a1.sinks.k1.type = org.example.MySink
    a1.sinks.k1.channel = c1