PostgreSQL数据库——Pigsty pg_exporter

男娘i 2022-09-14 00:19 291阅读 0赞

## Prometheus架构和生态 ##

![在这里插入图片描述][watermark_type_ZHJvaWRzYW5zZmFsbGJhY2s_shadow_50_text_Q1NETiBA6IKl5Y-U6I-M_size_20_color_FFFFFF_t_70_g_se_x_16]  
组件说明

*  Prometheus server : 用于抓取和存储时间序列数据
 *  用于检测应用程序代码的客户端库
 *  push gateway:支持短时任务
 *  针对特定服务的Exporter,如HAProxy, StatsD, Graphite等
 *  一个处理告警的告警管理器
 *  各种工具  
    Prometheus **直接或者间接通过gateway拉去指标度量值**，并将拉去的样本存储在本地。从拉去的样本数据中按照规则聚合并产生新的时序数据，或者告警数据。通过Grafana将收集的数据可视化。

## 如何为中间件开发Exporter ##

Prometheus 为开发这提供了客户端工具，用于为自己的中间件开发Exporter，对接Prometheus。目前支持的客户端Go、Java、Python、Ruby。

### 依赖包的引入 ###

工程结构
    [root@node1 data]# tree exporter/
    exporter/
    ├── collector
    │   └── node.go
    ├── go.mod
    └── main.go
    1 directory, 3 files
    引入依赖包
    require (
        github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
        github.com/modern-go/reflect2 v1.0.1 // indirect
        github.com/prometheus/client_golang v1.1.0
            //借助gopsutil 采集主机指标
        github.com/shirou/gopsutil v0.0.0-20190731134726-d80c43f9c984
    )

main.go

package main
    import (
        "cloud.io/exporter/collector"
        "fmt"
        "github.com/prometheus/client_golang/prometheus"
        "github.com/prometheus/client_golang/prometheus/promhttp"
        "net/http"
    )
    func init()  { 
         //注册自身采集器
        prometheus.MustRegister(collector.NewNodeCollector())
    }
    func main() { 
        http.Handle("/metrics", promhttp.Handler())
        if err := http.ListenAndServe(":8080", nil); err != nil { 
            fmt.Printf("Error occur when start server %v", err)
        }
    }

为了能看清结果我将默认采集器注释,位置registry.go

func init() { 
        //MustRegister(NewProcessCollector(ProcessCollectorOpts{}))
        //MustRegister(NewGoCollector())
    }

/collector/node.go 代码中涵盖了Counter、Gauge、Histogram、Summary四种情况，一起混合使用的情况，具体的说明见一下代码中。

package collector
    import (
        "github.com/prometheus/client_golang/prometheus"
        "github.com/shirou/gopsutil/host"
        "github.com/shirou/gopsutil/mem"
        "runtime"
        "sync"
    )
    var reqCount int32
    var hostname string
    type NodeCollector struct { 
        requestDesc    *prometheus.Desc   //Counter
        nodeMetrics     nodeStatsMetrics  //混合方式 
        goroutinesDesc *prometheus.Desc   //Gauge
        threadsDesc    *prometheus.Desc  //Gauge
        summaryDesc    *prometheus.Desc  //summary
        histogramDesc  *prometheus.Desc   //histogram
        mutex          sync.Mutex
    }
    //混合方式数据结构
    type nodeStatsMetrics []struct { 
        desc    *prometheus.Desc
        eval    func(*mem.VirtualMemoryStat) float64
        valType prometheus.ValueType
    }
    //初始化采集器
    func NewNodeCollector() prometheus.Collector { 
        host,_:= host.Info()
        hostname = host.Hostname
        return &NodeCollector{ 
            requestDesc: prometheus.NewDesc(
                "total_request_count",
                "请求数",
                []string{ "DYNAMIC_HOST_NAME"}, //动态标签名称
                prometheus.Labels{ "STATIC_LABEL1":"静态值可以放在这里","HOST_NAME":hostname}),
            nodeMetrics: nodeStatsMetrics{ 
                { 
                    desc: prometheus.NewDesc(
                        "total_mem",
                        "内存总量",
                        nil, nil),
                    valType: prometheus.GaugeValue,
                    eval: func(ms *mem.VirtualMemoryStat) float64 {  return float64(ms.Total) / 1e9 },
                },
                { 
                    desc: prometheus.NewDesc(
                        "free_mem",
                        "内存空闲",
                        nil, nil),
                    valType: prometheus.GaugeValue,
                    eval: func(ms *mem.VirtualMemoryStat) float64 {  return float64(ms.Free) / 1e9 },
                },
            },
            goroutinesDesc:prometheus.NewDesc(
                "goroutines_num",
                "协程数.",
                nil, nil),
            threadsDesc: prometheus.NewDesc(
                "threads_num",
                "线程数",
                nil, nil),
            summaryDesc: prometheus.NewDesc(
                "summary_http_request_duration_seconds",
                "summary类型",
                []string{ "code", "method"},
                prometheus.Labels{ "owner": "example"},
            ),
            histogramDesc: prometheus.NewDesc(
                "histogram_http_request_duration_seconds",
                "histogram类型",
                []string{ "code", "method"},
                prometheus.Labels{ "owner": "example"},
            ),
        }
    }
    // Describe returns all descriptions of the collector.
    //实现采集器Describe接口
    func (n *NodeCollector) Describe(ch chan<- *prometheus.Desc) { 
        ch <- n.requestDesc
        for _, metric := range n.nodeMetrics { 
            ch <- metric.desc
        }
        ch <- n.goroutinesDesc
        ch <- n.threadsDesc
        ch <- n.summaryDesc
        ch <- n.histogramDesc
    }
    // Collect returns the current state of all metrics of the collector.
    //实现采集器Collect接口,真正采集动作
    func (n *NodeCollector) Collect(ch chan<- prometheus.Metric) { 
        n.mutex.Lock()
        ch <- prometheus.MustNewConstMetric(n.requestDesc,prometheus.CounterValue,0,hostname)
        vm, _ := mem.VirtualMemory()
        for _, metric := range n.nodeMetrics { 
            ch <- prometheus.MustNewConstMetric(metric.desc, metric.valType, metric.eval(vm))
        }
        ch <- prometheus.MustNewConstMetric(n.goroutinesDesc, prometheus.GaugeValue, float64(runtime.NumGoroutine()))
        num, _ := runtime.ThreadCreateProfile(nil)
        ch <- prometheus.MustNewConstMetric(n.threadsDesc, prometheus.GaugeValue, float64(num))
        //模拟数据
        ch <- prometheus.MustNewConstSummary(
            n.summaryDesc,
            4711, 403.34,
            map[float64]float64{ 0.5: 42.3, 0.9: 323.3},
            "200", "get",
        )
        //模拟数据
        ch <- prometheus.MustNewConstHistogram(
                n.histogramDesc,
                4711, 403.34,
                map[float64]uint64{ 25: 121, 50: 2403, 100: 3221, 200: 4233},
                "200", "get",
            )
        n.mutex.Unlock()
    }

执行的结果http://127.0.0.1:8080/metrics  
![在这里插入图片描述][watermark_type_ZHJvaWRzYW5zZmFsbGJhY2s_shadow_50_text_Q1NETiBA6IKl5Y-U6I-M_size_20_color_FFFFFF_t_70_g_se_x_16 1]

## pg\_exporter ##

https://api.github.com/repos/Vonng/pg\_exporter/releases/latest

[Prometheus](https://prometheus.io/) [exporter](https://prometheus.io/docs/instrumenting/exporters/) for [PostgreSQL](https://www.postgresql.org) metrics. **Gives you complete insight on your favourate elephant!**
    PG Exporter is the foundation component for Project [Pigsty](https://pigsty.cc), Which maybe the best **OpenSource** Monitoring Solution for PostgreSQL.
    Latest binaries & rpms can be found on [release](https://github.com/Vonng/pg_exporter/releases) page. Supported pg version: PostgreSQL 9.4+ & Pgbouncer 1.8+. Default collectors definition is compatible with PostgreSQL 10,11,12,13,14. 
    Latest stable `pg_exporter` version is: `0.3.2` , and latest beta `pg_exporter` version is: `0.4.0beta` .

如何运行

*  **Where** to scrape: A postgres or pgbouncer URL given via `PG_EXPORTER_URL` or `--url`
 *  **What** to scrape: A path to config file or directory, by default `./pg_exporter.yaml` or `/etc/pg_exporter`

export PG_EXPORTER_URL='postgres://postgres:password@localhost:5432/postgres'
    export PG_EXPORTER_CONFIG='/path/to/conf/file/or/dir'
    pg_exporter

## Run ##

Parameter could be given via command line arguments or environment variables.

usage: pg_exporter [<flags>]
    
    Flags:
      --help                        Show context-sensitive help (also try --help-long and --help-man).
      --url=URL                     postgres target url
      --config=CONFIG               path to config dir or file
      --label=""                    constant lables:comma separated list of label=value pair
      --tag=""                      tags,comma separated list of server tag
      --disable-cache               force not using cache
      --disable-intro               disable collector level introspection metrics
      --auto-discovery              automatically scrape all database for given server
      --exclude-database="template0,template1,postgres"
                                    excluded databases when enabling auto-discovery
      --include-database=""         included databases when enabling auto-discovery
      --namespace=""                prefix of built-in metrics, (pg|pgbouncer) by default
      --fail-fast                   fail fast instead of waiting during start-up
      --web.listen-address=":9630"  prometheus web server listen address
      --web.telemetry-path="/metrics"
                                    URL path under which to expose metrics.
      --dry-run                     dry run and print raw configs
      --explain                     explain server planned queries
      --version                     Show application version.
      --log.level="info"            Only log messages with the given severity or above. Valid levels: [debug, info, warn, error, fatal]
      --log.format="logger:stderr"  Set the log target and format. Example: "logger:syslog?appname=bob&local=7" or "logger:stdout?json=true"

## API ##

Here are `pg_exporter` REST APIs

# Fetch metrics (metrics path depends on parameters)
    curl localhost:9630/metrics
    # Reload configuration
    curl localhost:9630/reload
    # Explain configuration
    curl localhost:9630/explain
    # Aliveness health check (200 up, 503 down)
    curl localhost:9630/up
    curl localhost:9630/health
    curl localhost:9630/liveness
    curl localhost:9630/readiness
    # traffic route health check
    ### 200 if not in recovery, 404 if in recovery, 503 if server is down
    curl localhost:9630/primary
    curl localhost:9630/leader
    curl localhost:9630/master
    curl localhost:9630/read-write
    curl localhost:9630/rw
    ### 200 if in recovery, 404 if not in recovery, 503 if server is down
    curl localhost:9630/replica
    curl localhost:9630/standby
    curl localhost:9630/slave
    curl localhost:9630/read-only
    curl localhost:9630/ro
    ### 200 if server is ready for read traffic (including primary), 503 if server is down
    curl localhost:9630/read

## 主函数 ##

func Run() { 
    	ParseArgs()
    	// explain config only
    	if *dryRun { 
    		DryRun()
    	}
    	if *configPath == "" { 
    		log.Errorf("no valid config path, exit")
    		os.Exit(1)
    	}
    
    	// DummyServer will server a constant pg_up
    	// launch a dummy server to check listen address availability
    	// and fake a pg_up 0 metrics before PgExporter connecting to target instance
    	// otherwise, exporter API is not available until target instance online
    	dummySrv, closeChan := DummyServer()
    
    	// create exporter: if target is down, exporter creation will wait until it backup online
    	var err error
    	PgExporter, err = NewExporter(
    		*pgURL,
    		WithConfig(*configPath),
    		WithConstLabels(*constLabels),
    		WithCacheDisabled(*disableCache),
    		WithFailFast(*failFast),
    		WithNamespace(*exporterNamespace),
    		WithAutoDiscovery(*autoDiscovery),
    		WithExcludeDatabase(*excludeDatabase),
    		WithIncludeDatabase(*includeDatabase),
    		WithTags(*serverTags),
    	)
    	if err != nil { 
    		log.Fatalf("fail creating pg_exporter: %s", err.Error())
    		os.Exit(2)
    	}
    
    	// trigger a manual planning before explain
    	if *explainOnly { 
    		PgExporter.server.Plan()
    		fmt.Println(PgExporter.Explain())
    		os.Exit(0)
    	}
    
    	prometheus.MustRegister(PgExporter)
    	defer PgExporter.Close()
    
    	// reload conf when receiving SIGHUP or SIGUSR1
    	sigs := make(chan os.Signal, 1)
    	signal.Notify(sigs, syscall.SIGHUP)
    	go func() { 
    		for sig := range sigs { 
    			switch sig { 
    			case syscall.SIGHUP:
    				log.Infof("%v received, reloading", sig)
    				_ = Reload()
    			}
    		}
    	}()
    
    	/*************** REST API ***************/
    	// basic
    	http.HandleFunc("/", TitleFunc)
    	http.HandleFunc("/version", VersionFunc)
    	// reload
    	http.HandleFunc("/reload", ReloadFunc)
    	// explain
    	http.HandleFunc("/explain", PgExporter.ExplainFunc)
    	// alive
    	http.HandleFunc("/up", PgExporter.UpCheckFunc)
    	http.HandleFunc("/read", PgExporter.UpCheckFunc)
    	http.HandleFunc("/health", PgExporter.UpCheckFunc)
    	http.HandleFunc("/liveness", PgExporter.UpCheckFunc)
    	http.HandleFunc("/readiness", PgExporter.UpCheckFunc)
    	// primary
    	http.HandleFunc("/primary", PgExporter.PrimaryCheckFunc)
    	http.HandleFunc("/leader", PgExporter.PrimaryCheckFunc)
    	http.HandleFunc("/master", PgExporter.PrimaryCheckFunc)
    	http.HandleFunc("/read-write", PgExporter.PrimaryCheckFunc)
    	http.HandleFunc("/rw", PgExporter.PrimaryCheckFunc)
    	// replica
    	http.HandleFunc("/replica", PgExporter.ReplicaCheckFunc)
    	http.HandleFunc("/standby", PgExporter.ReplicaCheckFunc)
    	http.HandleFunc("/slave", PgExporter.ReplicaCheckFunc)
    	http.HandleFunc("/read-only", PgExporter.ReplicaCheckFunc)
    	http.HandleFunc("/ro", PgExporter.ReplicaCheckFunc)
    
    	// metric
    	_ = dummySrv.Close()
    	<-closeChan
    	http.Handle(*metricPath, promhttp.Handler())
    
    	log.Infof("pg_exporter for %s start, listen on http://%s%s", shadowDSN(*pgURL), *listenAddress, *metricPath)
    	log.Fatal(http.ListenAndServe(*listenAddress, nil))
    }

[watermark_type_ZHJvaWRzYW5zZmFsbGJhY2s_shadow_50_text_Q1NETiBA6IKl5Y-U6I-M_size_20_color_FFFFFF_t_70_g_se_x_16]: /images/20220828/9f03c1900a5d4a6c94fb8d6965981389.png
[watermark_type_ZHJvaWRzYW5zZmFsbGJhY2s_shadow_50_text_Q1NETiBA6IKl5Y-U6I-M_size_20_color_FFFFFF_t_70_g_se_x_16 1]: /images/20220828/d5c5f496d7e74b96985d38deb848731e.png