From 55c33c77ddd317dd5a0b3764a5849d01dfe73fd9 Mon Sep 17 00:00:00 2001
From: tracedeng <56997132@qq.com>
Date: Mon, 16 Aug 2021 10:05:31 +0800
Subject: [PATCH] obs build statistics to grafana
---
changelogs/changelog_03/fluent.conf | 222 ++++++++++++++++++
.../obs_build_statistics_to_grafana.md | 75 ++++++
changelogs/openEuler_OBS_changelogs.md | 2 +-
3 files changed, 298 insertions(+), 1 deletion(-)
create mode 100644 changelogs/changelog_03/fluent.conf
create mode 100644 changelogs/changelog_03/obs_build_statistics_to_grafana.md
diff --git a/changelogs/changelog_03/fluent.conf b/changelogs/changelog_03/fluent.conf
new file mode 100644
index 0000000..2c9994d
--- /dev/null
+++ b/changelogs/changelog_03/fluent.conf
@@ -0,0 +1,222 @@
+# openEuler:Mainline
+
+ @type tail
+ path /srv/obs/build/openEuler:Mainline/standard_x86_64/x86_64/:jobhistory,/srv/obs/build/openEuler:Mainline/standard_aarch64/aarch64/:jobhistory
+ pos_file /srv/obs/build/fluentd/pos_file/openEuler:Mainline
+ tag obs_job_history.openEuler:Mainline
+ refresh_interval 60s
+ #read_from_head true
+
+ @type regexp
+ expression /^(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)$/
+ types readytime:integer, starttime:integer, endtime:integer
+
+
+
+# openEuler:Factory
+
+ @type tail
+ path /srv/obs/build/openEuler:Factory/standard_x86_64/x86_64/:jobhistory,/srv/obs/build/openEuler:Factory/standard_aarch64/aarch64/:jobhistory
+ pos_file /srv/obs/build/fluentd/pos_file/openEuler:Factory
+ tag obs_job_history.openEuler:Factory
+ refresh_interval 60s
+ #read_from_head true
+
+ @type regexp
+ expression /^(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)$/
+ types readytime:integer, starttime:integer, endtime:integer
+
+
+
+# openEuler:20.03:LTS
+
+ @type tail
+ path /srv/obs/build/openEuler:20.03:LTS/standard_x86_64/x86_64/:jobhistory,/srv/obs/build/openEuler:20.03:LTS/standard_aarch64/aarch64/:jobhistory
+ pos_file /srv/obs/build/fluentd/pos_file/openEuler:20.03:LTS
+ tag obs_job_history.openEuler:20.03:LTS
+ refresh_interval 60s
+ #read_from_head true
+
+ @type regexp
+ expression /^(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)$/
+ types readytime:integer, starttime:integer, endtime:integer
+
+
+
+# openEuler:20.03:NEXT
+
+ @type tail
+ path /srv/obs/build/openEuler:20.03:LTS:Next/standard_x86_64/x86_64/:jobhistory,/srv/obs/build/openEuler:20.03:LTS:Next/standard_aarch64/aarch64/:jobhistory
+ pos_file /srv/obs/build/fluentd/pos_file/openEuler:20.03:LTS:Next
+ tag obs_job_history.openEuler:20.03:LTS:Next
+ refresh_interval 60s
+ #read_from_head true
+
+ @type regexp
+ expression /^(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)$/
+ types readytime:integer, starttime:integer, endtime:integer
+
+
+
+# openEuler:20.03:LTS:SP1
+
+ @type tail
+ path /srv/obs/build/openEuler:20.03:LTS:SP1/standard_x86_64/x86_64/:jobhistory,/srv/obs/build/openEuler:20.03:LTS:SP1/standard_aarch64/aarch64/:jobhistory
+ pos_file /srv/obs/build/fluentd/pos_file/openEuler:20.03:LTS:SP1
+ tag obs_job_history.openEuler:20.03:LTS:SP1
+ refresh_interval 60s
+ #read_from_head true
+
+ @type regexp
+ expression /^(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)$/
+ types readytime:integer, starttime:integer, endtime:integer
+
+
+
+# openEuler:20.03:LTS:SP2
+
+ @type tail
+ path /srv/obs/build/openEuler:20.03:LTS:SP2/standard_x86_64/x86_64/:jobhistory,/srv/obs/build/openEuler:20.03:LTS:SP2/standard_aarch64/aarch64/:jobhistory
+ pos_file /srv/obs/build/fluentd/pos_file/openEuler:20.03:LTS:SP2
+ tag obs_job_history.openEuler:20.03:LTS:SP2
+ refresh_interval 60s
+ read_from_head true
+
+ @type regexp
+ expression /^(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)$/
+ types readytime:integer, starttime:integer, endtime:integer
+
+
+
+# openEuler:20.09
+
+ @type tail
+ path /srv/obs/build/openEuler:20.09/standard_x86_64/x86_64/:jobhistory,/srv/obs/build/openEuler:20.09/standard_aarch64/aarch64/:jobhistory
+ pos_file /srv/obs/build/fluentd/pos_file/openEuler:20.09
+ tag obs_job_history.openEuler:20.09
+ refresh_interval 60s
+ #read_from_head true
+
+ @type regexp
+ expression /^(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)$/
+ types readytime:integer, starttime:integer, endtime:integer
+
+
+
+# openEuler:21.03
+
+ @type tail
+ path /srv/obs/build/openEuler:21.03/standard_x86_64/x86_64/:jobhistory,/srv/obs/build/openEuler:21.03/standard_aarch64/aarch64/:jobhistory
+ pos_file /srv/obs/build/fluentd/pos_file/openEuler:21.03
+ tag obs_job_history.openEuler:21.03
+ refresh_interval 60s
+ read_from_head true
+
+ @type regexp
+ expression /^(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)$/
+ types readytime:integer, starttime:integer, endtime:integer
+
+
+
+# openEuler:21.09
+
+ @type tail
+ path /srv/obs/build/openEuler:21.09/standard_x86_64/x86_64/:jobhistory,/srv/obs/build/openEuler:21.09/standard_aarch64/aarch64/:jobhistory
+ pos_file /srv/obs/build/fluentd/pos_file/openEuler:21.09
+ tag obs_job_history.openEuler:21.09
+ refresh_interval 60s
+ read_from_head true
+
+ @type regexp
+ expression /^(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)$/
+ types readytime:integer, starttime:integer, endtime:integer
+
+
+
+
+ enable_msgpack_time_support true
+
+
+# filter
+
+ @type record_transformer
+ enable_ruby
+
+ project ${tag_suffix[1]}
+ duration ${record["endtime"] - record["starttime"]}
+ pending ${record["starttime"] - record["readytime"]}
+ success ${if record["code"] == "succeeded"; 1; else; 0; end;}
+ failed ${if record["code"] == "succeeded"; 0; else; 1; end;}
+ readytime_date ${ require 'date'; DateTime.strptime(record["readytime"].to_s, "%s").new_offset(8.0/24).strftime("%Y-%m-%dT%H:%M:%S%z") }
+ starttime_date ${ require 'date'; DateTime.strptime(record["starttime"].to_s, "%s").new_offset(8.0/24).strftime("%Y-%m-%dT%H:%M:%S%z") }
+ endtime_date ${ require 'date'; DateTime.strptime(record["endtime"].to_s, "%s").new_offset(8.0/24).strftime("%Y-%m-%dT%H:%M:%S%z") }
+ created_at ${ require 'date'; DateTime.strptime(record["starttime"].to_s, "%s").new_offset(8.0/24).strftime("%Y-%m-%dT%H:%M:%S%z") }
+
+
+
+# route
+
+ @type kafka2
+ brokers 159.138.58.212:9092,159.138.58.212:9093,159.138.58.212:9094
+
+
+ @type json
+
+
+
+ @type file
+ path /srv/obs/build/fluentd/buff_file/buffer
+ flush_interval 3s
+ chunk_limit_size 5m
+
+
+ topic_key topic
+ default_topic openeuler_statewall_ci_obs_job_history
+
+ require_ack 1
+
+
+# obs build status summary
+
+ @type tail
+ path /srv/obs/build/fluentd/build_status_summary
+ pos_file /srv/obs/build/fluentd/pos_file/build_status_summary
+ tag obs_build_status.summary
+ refresh_interval 60s
+ read_from_head true
+
+ @type regexp
+ expression /^(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)$/
+ types ctime:integer, building:integer, blocked:integer, signing:integer, finished:integer, scheduled:integer, dispatching:integer, succeeded:integer, failed:integer, unresolvable:integer, disabled:integer, excluded:integer, broken:integer, locked:integer, unknown:integer, ctime:integer, stable:integer
+
+
+
+
+ @type record_transformer
+ enable_ruby
+
+ runing ${record["building"] + record["blocked"] + record["signing"] + record["finished"] + record["scheduled"] + record["dispatching"]}
+ foldup ${record["succeeded"] + record["failed"] + record["unresolvable"] + record["disabled"] + record["excluded"] + record["broken"] + record["locked"] + record["unknown"]}
+ ctime_date ${ require 'date'; DateTime.strptime(record["ctime"].to_s, "%s").new_offset(8.0/24).strftime("%Y-%m-%dT%H:%M:%S%z") }
+
+
+
+
+ @type kafka2
+ brokers 159.138.58.212:9092,159.138.58.212:9093,159.138.58.212:9094
+
+
+ @type json
+
+
+
+ @type file
+ path /srv/obs/build/fluentd/buff_file/buffer2
+ flush_interval 3s
+ chunk_limit_size 5m
+
+ topic_key topic2
+ default_topic openeuler_statewall_ci_obs_build_status_summary
+
+ required_acks 1
+
diff --git a/changelogs/changelog_03/obs_build_statistics_to_grafana.md b/changelogs/changelog_03/obs_build_statistics_to_grafana.md
new file mode 100644
index 0000000..79f3030
--- /dev/null
+++ b/changelogs/changelog_03/obs_build_statistics_to_grafana.md
@@ -0,0 +1,75 @@
+# obs构建历史数据同步
+## 安装fluentd
+```shell script
+yum install ruby
+yum install ruby-dev
+yum install gem
+
+# 安装不上可尝试修改ruby源
+# gem source -a https://gems.ruby-china.com
+
+gem install fluentd
+gem install json
+
+fluent-gem install fluent-plugin-kafka
+```
+
+````shell script
+# 目录下生产配置文件/etc/fluentd/fluent.conf
+fluentd --setup /etc/fluentd
+````
+
+## fluent配置
+
+> fluent.conf
+
+```
+# openEuler:Mainline
+
+ @type tail
+ path /srv/obs/build/openEuler:Mainline/standard_x86_64/x86_64/:jobhistory,/srv/obs/build/openEuler:Mainline/standard_aarch64/aarch64/:jobhistory
+ pos_file /srv/obs/build/fluentd_pos_file/openEuler:Mainline
+ tag obs_job_history.openEuler:Mainline
+ refresh_interval 60s
+ read_from_head true
+
+ @type regexp
+ expression /^(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)\|(?[^|]*)$/
+
+
+
+# filter
+
+ @type record_transformer
+
+ project ${tag_parts[1]}
+ duration ${record["endtime"] - record["starttime"]}
+ fluentd_time ${time}
+
+
+
+# route
+
+ @type kafka2
+ brokers 127.0.0.1:9092
+
+
+ @type json
+
+
+
+ @type file
+ path /srv/obs/build/fulentd/fluentd_buffer
+ flush_interval 3s
+
+ topic_key topic
+ default_topic openeuler_statewall_obs_job_history
+
+ required_acks 1
+
+```
+
+## 运行fluentd
+```shell script
+fluentd -c /etc/fluentd/fluentd.conf -d /var/log/fluentd/fluentd.pid -o /var/log/fluentd/fluentd.log
+```
diff --git a/changelogs/openEuler_OBS_changelogs.md b/changelogs/openEuler_OBS_changelogs.md
index 47ac756..98e52ce 100644
--- a/changelogs/openEuler_OBS_changelogs.md
+++ b/changelogs/openEuler_OBS_changelogs.md
@@ -20,7 +20,7 @@
|--|--|--|--|--|--|--|--|
| 1 | 2021-08-06 | 硬件扩展|OBS构建时长优化（测试）|新增other_backend节点。该节点新增完成后用于配置openEuler-Mainline-copy工程。加入ARM 002、003、004等三个worker节点到新server（other_backend）|曹志 |否|NULL|
| 2 | 2021-08-12 | 配置变更|OBS构建时长优化（测试）|原来注册到backend节点的说有worker节点都注册到给other_back的server节点上。节点信息：X86:obs-worker-001,obs-worker-002,obs-worker-003,obs-worker-004,obs-worker-005,obs-worker-006,obs-worker-007,obs-worker-008,obs-worker-009,obs-worker-010,obs-worker-0010,obs-worker-0011,obs-worker-0012,obs-worker-0013,obs-worker-0018,obs-worker-0019,obs-worker-0020,obs-worker-0021,obs-worker-0022,obs-worker-0023,obs-worker-0024,obs-worker-0025,obs-worker-0026,obs-worker-0027,obs-worker-0028,obs-worker-0029,obs-worker-0030,ARM:ecs-obsworker-201,ecs-obsworker-202,ecs-obsworker-203,ecs-obsworker-204,ecs-obsworker-205,ecs-obsworker-206,ecs-obsworker-207,ecs-obsworker-208,ecs-obsworker-209,ecs-obsworker-0001,ecs-obsworker-0002,ecs-obsworker-0003,ecs-obsworker-0004,ecs-obsworker-0005,ecs-obsworker-0006,ecs-obsworker-0007,ecs-obsworker-0008,ecs-obsworker-0013,ecs-obsworker-0014,ecs-obsworker-0015,ecs-obsworker-0016,ecs-obsworker-0017,ecs-obsworker-0018,ecs-obsworker-0019,ecs-obsworker-0020|陈燕潘 |否|NULL|
-| 3 | | | | | | | |
+| 3 | 20210816 | 常驻服务 | 看板数据（单包构建时长） | 参考 "obs_build_statistics_to_grafana.md" | 邓鹏 | 是 | changelog_03/fluent.conf, changelog_03/obs_build_statistics_to_grafana.md|
| 4 | | | | | | | |
--
Gitee