convert falcon data to exporters for prometheus scraping (open-falcon#959)

Co-authored-by: makaijun <[email protected]>

Author: BruceMa29

Makefile

@@ -1,4 +1,4 @@
-CMD = agent aggregator graph hbs judge nodata transfer gateway api alarm
+CMD = agent aggregator graph hbs judge nodata transfer gateway api alarm falcon2p8s
 TARGET = open-falcon
 PACKAGES ?= $(shell go list ./... | grep -v /vendor/)
 GOFILES := $(shell find . -name "*.go" -type f -not -path "./vendor/*")

common/model/prometheus.go

@@ -0,0 +1,12 @@
+package model
+
+// MetricType Prometheus指标类型：GAUGE|COUNTER
+type P8sItem struct {
+	Endpoint   string            `json:"endpoint"`
+	Metric     string            `json:"metric"`
+	Tags       map[string]string `json:"tags"`
+	Value      float64           `json:"value"`
+	Timestamp  int64             `json:"timestamp"`
+	MetricType string            `json:"metric_type"`
+	Step       int               `json:"step"`
+}

config/falcon2p8s.json

@@ -0,0 +1,10 @@
+{
+    "log_level": "debug",
+    "concurrent": 100,
+    "http": {
+        "listen": "0.0.0.0:9090"
+    },
+    "rpc": {
+        "listen": "0.0.0.0:8080"
+    }
+}

config/transfer.json

@@ -71,5 +71,22 @@
         "db": "foo",
         "address": "http://127.0.0.1:8086",
         "timeout": 5000
+    },
+    "p8s_relay": {
+        "enabled": false,
+        "batch": 200,
+        "connTimeout": 1000,
+        "callTimeout": 5000,
+        "maxConns": 32,
+        "maxIdle": 32,
+        "replicas": 500,
+        "notSyncMetrics": [
+            "gpu.",
+            "snmp."
+        ],
+        "cluster": {
+            "p8s-relay-00" : "192.168.1.1:8080",
+            "p8s-relay-01" : "192.168.1.2:8080"
+        }
     }
 }

g/g.go

@@ -36,6 +36,7 @@ func init() {
 		"gateway":    true,
 		"api":        true,
 		"alarm":      true,
+		"falcon2p8s": true,
 	}
 
 	BinOf = map[string]string{
@@ -49,6 +50,7 @@ func init() {
 		"gateway":    "./gateway/bin/falcon-gateway",
 		"api":        "./api/bin/falcon-api",
 		"alarm":      "./alarm/bin/falcon-alarm",
+		"falcon2p8s": "./falcon2p8s/bin/falcon-falcon2p8s",
 	}
 
 	cfgOf = map[string]string{
@@ -62,6 +64,7 @@ func init() {
 		"gateway":    "./gateway/config/cfg.json",
 		"api":        "./api/config/cfg.json",
 		"alarm":      "./alarm/config/cfg.json",
+		"falcon2p8s": "./falcon2p8s/config/cfg.json",
 	}
 
 	ModuleApps = map[string]string{
@@ -75,6 +78,7 @@ func init() {
 		"gateway":    "falcon-gateway",
 		"api":        "falcon-api",
 		"alarm":      "falcon-alarm",
+		"falcon2p8s": "falcon-falcon2p8s",
 	}
 
 	logpathOf = map[string]string{
@@ -88,6 +92,7 @@ func init() {
 		"gateway":    "./gateway/logs/gateway.log",
 		"api":        "./api/logs/api.log",
 		"alarm":      "./alarm/logs/alarm.log",
+		"falcon2p8s": "./falcon2p8s/logs/falcon2p8s.log",
 	}
 
 	PidOf = map[string]string{
@@ -101,6 +106,7 @@ func init() {
 		"gateway":    "<NOT SET>",
 		"api":        "<NOT SET>",
 		"alarm":      "<NOT SET>",
+		"falcon2p8s": "<NOT SET>",
 	}
 
 	// Modules are deployed in this order
@@ -115,6 +121,7 @@ func init() {
 		"gateway",
 		"api",
 		"alarm",
+		"falcon2p8s",
 	}
 }
 

go.mod

@@ -28,7 +28,9 @@ require (
 	github.com/niean/go-metrics-lite v0.0.0-20151230091537-b5d30971b578 // indirect
 	github.com/niean/goperfcounter v0.0.0-20160108100052-24860a8d3fac
 	github.com/niean/gotools v0.0.0-20151221085310-ff3f51fc5c60 // indirect
+	github.com/oleiade/lane v1.0.1
 	github.com/open-falcon/rrdlite v0.0.0-20170412122036-7d8646c85cc5
+	github.com/prometheus/client_golang v1.5.1
 	github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b
 	github.com/sirupsen/logrus v1.8.1
 	github.com/smartystreets/goconvey v1.6.4

go.sum

@@ -102,6 +102,7 @@ github.com/benbjohnson/immutable v0.2.1/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylH
 github.com/benbjohnson/tmpl v1.0.0/go.mod h1:igT620JFIi44B6awvU9IsDhR77IXWtFigTLil/RPdps=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
@@ -624,6 +625,8 @@ github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtb
 github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
 github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
+github.com/oleiade/lane v1.0.1 h1:hXofkn7GEOubzTwNpeL9MaNy8WxolCYb9cInAIeqShU=
+github.com/oleiade/lane v1.0.1/go.mod h1:IyTkraa4maLfjq/GmHR+Dxb4kCMtEGeb+qmhlrQ5Mk4=
 github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=

modules/falcon2p8s/.gitignore

@@ -0,0 +1,7 @@
+*.log
+.idea
+.DS_Store
+/var
+/falcon2p8s*
+/cfg.json
+/gitversion

modules/falcon2p8s/ReadMe.md

@@ -0,0 +1,59 @@
+# falcon2p8s
+
+## 该组件是一个prometheus exporter，实现了将falcon监控数据转储到prometheus
+
+
+## 源码编译
+
+```bash
+cd $GOPATH/src/github.com/open-falcon
+git clone https://github.com/open-falcon/falcon-plus.git
+cd falcon-plus/modules/falcon2p8s
+go get 
+./control build
+./control pack
+```
+
+最后一步会pack出一个tar.gz的安装包，拿着这个包去部署服务即可。
+
+## 服务部署
+服务部署，包括配置修改、启动服务、检验服务、停止服务等。这之前，需要将安装包解压到服务的部署目录下。
+
+```bash
+# 修改配置, 配置项含义见下文
+mv cfg.example.json cfg.json
+vim cfg.json
+
+# 启动服务
+./control start
+
+# 校验服务，看端口是否在监听
+ss -tln
+
+# 检查log
+./control tail
+
+...
+# 停止服务
+./control stop
+
+```
+
+## 配置说明
+配置文件默认为./cfg.json。默认情况下，安装包会有一个cfg.example.json的配置文件示例。各配置项的含义，如下
+
+```bash
+## Configuration
+{
+    "log_level": "debug",
+    "concurrent": 100, # 处理metric的最大并发数
+	"http": {
+		"listen": "0.0.0.0:9090"
+	},
+    "rpc": {
+        "listen": "0.0.0.0:8080"
+    }
+}
+```
+
+sm.example.yaml是使用prometheus operator中servicemonitor做服务发现时，对应的yaml，作为参考

modules/falcon2p8s/cfg.example.json

@@ -0,0 +1,10 @@
+{
+    "log_level": "debug",
+    "concurrent": 100,
+	"http": {
+		"listen": "0.0.0.0:9090"
+	},
+    "rpc": {
+        "listen": "0.0.0.0:8080"
+    }
+}

modules/falcon2p8s/control

@@ -0,0 +1,113 @@
+#!/bin/bash
+
+WORKSPACE=$(cd $(dirname $0)/; pwd)
+cd $WORKSPACE
+
+mkdir -p var
+
+module=falcon2p8s
+app=falcon2p8s
+conf=cfg.json
+pidfile=var/app.pid
+logfile=var/app.log
+
+function check_pid() {
+    if [ -f $pidfile ];then
+        pid=`cat $pidfile`
+        if [ -n $pid ]; then
+            running=`ps -p $pid|grep -v "PID TTY" |wc -l`
+            return $running
+        fi
+    fi
+    return 0
+}
+
+function start() {
+    check_pid
+    running=$?
+    if [ $running -gt 0 ];then
+        echo -n "$app now is running already, pid="
+        cat $pidfile
+        return 1
+    fi
+
+    nohup ./$app -c $conf >> $logfile 2>&1 &
+    echo $! > $pidfile
+    echo "$app started..., pid=$!"
+}
+
+function stop() {
+    pid=`cat $pidfile`
+    kill $pid
+    echo "$app stopped..."
+}
+
+function restart() {
+    stop
+    sleep 1
+    start
+}
+
+function status() {
+    check_pid
+    running=$?
+    if [ $running -gt 0 ];then
+        echo -n "$app now is running, pid="
+        cat $pidfile
+    else
+        echo "$app is stopped"
+    fi
+}
+
+function tailf() {
+    tail -f $logfile
+}
+
+function build() {
+    go build
+    if [ $? -ne 0 ]; then
+        exit $?
+    fi
+    mv $module $app
+    ./$app -v
+}
+
+function pack() {
+    build
+    git log -1 --pretty=%h > gitversion
+    version=`./$app -v`
+    tar zcvf $app-$version.tar.gz control cfg.example.json $app gitversion
+}
+
+function packbin() {
+    build
+    git log -1 --pretty=%h > gitversion
+    version=`./$app -v`
+    tar zcvf $app-bin-$version.tar.gz $app gitversion
+}
+
+function help() {
+    echo "$0 build|pack|packbin|start|stop|restart|status|tail"
+}
+
+if [ "$1" == "" ]; then
+    help
+elif [ "$1" == "stop" ];then
+    stop
+elif [ "$1" == "start" ];then
+    start
+elif [ "$1" == "restart" ];then
+    restart
+elif [ "$1" == "status" ];then
+    status
+elif [ "$1" == "tail" ];then
+    tailf
+elif [ "$1" == "build" ];then
+    build
+elif [ "$1" == "pack" ];then
+    pack
+elif [ "$1" == "packbin" ];then
+    packbin
+else
+    help
+fi

modules/falcon2p8s/g/cfg.go

@@ -0,0 +1,64 @@
+package g
+
+import (
+	"encoding/json"
+	"sync"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/toolkits/file"
+)
+
+type HttpConfig struct {
+	Listen string `json:"listen"`
+}
+
+type RpcConfig struct {
+	Listen string `json:"listen"`
+}
+
+type GlobalConfig struct {
+	LogLevel   string      `json:"log_level"`
+	Concurrent int         `json:"concurrent"`
+	Http       *HttpConfig `json:"http"`
+	Rpc        *RpcConfig  `json:"rpc"`
+}
+
+var (
+	ConfigFile string
+	config     *GlobalConfig
+	configLock = new(sync.RWMutex)
+)
+
+func Config() *GlobalConfig {
+	configLock.RLock()
+	defer configLock.RUnlock()
+	return config
+}
+
+func ParseConfig(cfg string) {
+	if cfg == "" {
+		log.Fatalln("use -c to specify configuration file")
+	}
+
+	if !file.IsExist(cfg) {
+		log.Fatalln("config file:", cfg, "is not existent")
+	}
+
+	ConfigFile = cfg
+
+	configContent, err := file.ToTrimString(cfg)
+	if err != nil {
+		log.Fatalln("read config file:", cfg, "fail:", err)
+	}
+
+	var c GlobalConfig
+	err = json.Unmarshal([]byte(configContent), &c)
+	if err != nil {
+		log.Fatalln("parse config file:", cfg, "fail:", err)
+	}
+
+	configLock.Lock()
+	defer configLock.Unlock()
+	config = &c
+	log.Println("read config file:", cfg, "successfully")
+}