diff --git a/content/en/docs/A-Tune/appendixes.md b/content/en/docs/A-Tune/appendixes.md index 46f489cac3e98bcc418e368e7f442270d31a13fa..2d776555c04a00f5a7c56e5d8b503925019af32a 100644 --- a/content/en/docs/A-Tune/appendixes.md +++ b/content/en/docs/A-Tune/appendixes.md @@ -8,19 +8,13 @@ **Table 1** Terminology - - - - - @@ -28,3 +22,4 @@

Term

Description

workload_type

-

Workload type, which is used to identify a type of service with the same characteristics.

-

profile

+

profile

Set of optimization items and optimal parameter configuration.
+ diff --git a/content/en/docs/A-Tune/application-scenarios.md b/content/en/docs/A-Tune/application-scenarios.md index 924170be9956171063f74f8a718545db93e55907..600b6e3a71c9156ad954776ec502d053ef6d8ff9 100644 --- a/content/en/docs/A-Tune/application-scenarios.md +++ b/content/en/docs/A-Tune/application-scenarios.md @@ -41,10 +41,6 @@ You can use functions provided by A-Tune through the CLI client atune-adm. This - The **define**, **update**, **undefine**, **collection**, **train**, and **upgrade **commands do not support remote execution. - In the command format, brackets \(\[\]\) indicate that the parameter is optional, and angle brackets \(<\>\) indicate that the parameter is mandatory. The actual parameters prevail. -- In the command format, meanings of each command are as follows: - - **WORKLOAD\_TYPE**: name of a user-defined workload type. For details about the supported workload types, see the query result of the **list** command. - - **PROFILE\_NAME**: user-defined profile name. - - **PROFILE\_PATH**: path of the user-defined profile. ## Querying Workload Types @@ -66,33 +62,107 @@ Query the supported workload types, profiles, and the values of Active. ``` # atune-adm list -Support WorkloadTypes: -+-----------------------------------+------------------------+-----------+ -| WorkloadType | ProfileName | Active | -+===================================+========================+===========+ -| default | default | true | -+-----------------------------------+------------------------+-----------+ -| webserver | ssl_webserver | false | -+-----------------------------------+------------------------+-----------+ -| big_database | database | false | -+-----------------------------------+------------------------+-----------+ -| big_data | big_data | false | -+-----------------------------------+------------------------+-----------+ -| in-memory_computing | in-memory_computing | false | -+-----------------------------------+------------------------+-----------+ -| in-memory_database | in-memory_database | false | -+-----------------------------------+------------------------+-----------+ -| single_computer_intensive_jobs | compute-intensive | false | -+-----------------------------------+------------------------+-----------+ -| communication | rpc_communication | false | -+-----------------------------------+------------------------+-----------+ -| idle | default | false | -+-----------------------------------+------------------------+-----------+ +Support profiles: ++------------------------------------------------+-----------+ +| ProfileName | Active | ++================================================+===========+ +| arm-native-android-container-robox | false | ++------------------------------------------------+-----------+ +| basic-test-suite-euleros-baseline-fio | false | ++------------------------------------------------+-----------+ +| basic-test-suite-euleros-baseline-lmbench | false | ++------------------------------------------------+-----------+ +| basic-test-suite-euleros-baseline-netperf | false | ++------------------------------------------------+-----------+ +| basic-test-suite-euleros-baseline-stream | false | ++------------------------------------------------+-----------+ +| basic-test-suite-euleros-baseline-unixbench | false | ++------------------------------------------------+-----------+ +| basic-test-suite-speccpu-speccpu2006 | false | ++------------------------------------------------+-----------+ +| basic-test-suite-specjbb-specjbb2015 | false | ++------------------------------------------------+-----------+ +| big-data-hadoop-hdfs-dfsio-hdd | false | ++------------------------------------------------+-----------+ +| big-data-hadoop-hdfs-dfsio-ssd | false | ++------------------------------------------------+-----------+ +| big-data-hadoop-spark-bayesian | false | ++------------------------------------------------+-----------+ +| big-data-hadoop-spark-kmeans | false | ++------------------------------------------------+-----------+ +| big-data-hadoop-spark-sql1 | false | ++------------------------------------------------+-----------+ +| big-data-hadoop-spark-sql10 | false | ++------------------------------------------------+-----------+ +| big-data-hadoop-spark-sql2 | false | ++------------------------------------------------+-----------+ +| big-data-hadoop-spark-sql3 | false | ++------------------------------------------------+-----------+ +| big-data-hadoop-spark-sql4 | false | ++------------------------------------------------+-----------+ +| big-data-hadoop-spark-sql5 | false | ++------------------------------------------------+-----------+ +| big-data-hadoop-spark-sql6 | false | ++------------------------------------------------+-----------+ +| big-data-hadoop-spark-sql7 | false | ++------------------------------------------------+-----------+ +| big-data-hadoop-spark-sql8 | false | ++------------------------------------------------+-----------+ +| big-data-hadoop-spark-sql9 | false | ++------------------------------------------------+-----------+ +| big-data-hadoop-spark-tersort | false | ++------------------------------------------------+-----------+ +| big-data-hadoop-spark-wordcount | false | ++------------------------------------------------+-----------+ +| cloud-compute-kvm-host | false | ++------------------------------------------------+-----------+ +| database-mariadb-2p-tpcc-c3 | false | ++------------------------------------------------+-----------+ +| database-mariadb-4p-tpcc-c3 | false | ++------------------------------------------------+-----------+ +| database-mongodb-2p-sysbench | false | ++------------------------------------------------+-----------+ +| database-mysql-2p-sysbench-hdd | false | ++------------------------------------------------+-----------+ +| database-mysql-2p-sysbench-ssd | false | ++------------------------------------------------+-----------+ +| database-postgresql-2p-sysbench-hdd | false | ++------------------------------------------------+-----------+ +| database-postgresql-2p-sysbench-ssd | false | ++------------------------------------------------+-----------+ +| default-default | false | ++------------------------------------------------+-----------+ +| docker-mariadb-2p-tpcc-c3 | false | ++------------------------------------------------+-----------+ +| docker-mariadb-4p-tpcc-c3 | false | ++------------------------------------------------+-----------+ +| hpc-gatk4-human-genome | false | ++------------------------------------------------+-----------+ +| in-memory-database-redis-redis-benchmark | false | ++------------------------------------------------+-----------+ +| middleware-dubbo-dubbo-benchmark | false | ++------------------------------------------------+-----------+ +| storage-ceph-vdbench-hdd | false | ++------------------------------------------------+-----------+ +| storage-ceph-vdbench-ssd | false | ++------------------------------------------------+-----------+ +| virtualization-consumer-cloud-olc | false | ++------------------------------------------------+-----------+ +| virtualization-mariadb-2p-tpcc-c3 | false | ++------------------------------------------------+-----------+ +| virtualization-mariadb-4p-tpcc-c3 | false | ++------------------------------------------------+-----------+ +| web-apache-traffic-server-spirent-pingpo | false | ++------------------------------------------------+-----------+ +| web-nginx-http-long-connection | true | ++------------------------------------------------+-----------+ +| web-nginx-https-short-connection | false | ++------------------------------------------------+-----------+ ``` >![](public_sys-resources/icon-note.gif) **NOTE:** ->If the value of Active is **true**, the profile is activated. In the example, the profile of the default type is activated. +>If the value of Active is **true**, the profile is activated. In the example, the profile of web-nginx-http-long-connection is activated. ## Workload Type Analysis and Auto Optimization @@ -121,7 +191,12 @@ Collect real-time statistics from the system to identify and automatically optim

--model, -m

-

Model generated by user-defined training

+

New model generated after user self-training

+ + +

--characterization, -c

+ +

Use the default model for application identification and do not perform automatic optimization

@@ -130,12 +205,18 @@ Collect real-time statistics from the system to identify and automatically optim #### Example -- Use the default model for classification and identification. +- Use the default model for application identification. ``` - # atune-adm analysis + # atune-adm analysis --characterization ``` +- Use the default model to identify applications and perform automatic tuning. + + ``` + # atune-adm analysis + ``` + - Use the user-defined training model for recognition. ``` @@ -147,8 +228,8 @@ Collect real-time statistics from the system to identify and automatically optim A-Tune allows users to define and learn new models. To define a new model, perform the following steps: -1. Run the **define** command to define workload\_type and profile. -2. Run the **collection** command to collect the profile data corresponding to workload\_type. +1. Run the **define** command to define a new profile. +2. Run the **collection** command to collect the system data corresponding to the application. 3. Run the **train** command to train the model. @@ -158,49 +239,47 @@ A-Tune allows users to define and learn new models. To define a new model, perfo #### Function -Add a user-defined workload type and the corresponding profile optimization item. +Add a user-defined application scenarios and the corresponding profile tuning items. #### Format -**atune-adm define** +**atune-adm define** #### Example -Add a workload type. Set workload type to **test\_type**, profile name to **test\_name**, and configuration file of an optimization item to **example.conf**. +Add a profile whose service_type is **test_service**, application_name is **test_app**, scenario_name is **test_scenario**, and tuning item configuration file is **example.conf**. ``` -# atune-adm define test_type test_name ./example.conf +# atune-adm define test_service test_app test_scenario ./example.conf ``` -The **example.conf** file can be written as follows \(the following optimization items are optional and are for reference only\). You can also run the **atune-adm info** command to view how the existing profile is written. +The **example.conf** file can be written as follows (the following optimization items are optional and are for reference only). You can also run the **atune-adm info** command to view how the existing profile is written. ``` -[main] -# list its parent profile -[tip] -# the recommended optimization, which should be performed manunaly -[check] -# check the environment -[affinity.irq] -# to change the affinity of irqs -[affinity.task] -# to change the affinity of tasks -[bios] -# to change the bios config -[bootloader.grub2] -# to change the grub2 config -[kernel_config] -# to change the kernel config -[script] -# the script extention of cpi -[sysctl] -# to change the /proc/sys/* config -[sysfs] -# to change the /sys/* config -[systemctl] -# to change the system service config -[ulimit] -# to change the resources limit of user + [main] + # list its parent profile + [kernel_config] + # to change the kernel config + [bios] + # to change the bios config + [bootloader.grub2] + # to change the grub2 config + [sysfs] + # to change the /sys/* config + [systemctl] + # to change the system service status + [sysctl] + # to change the /proc/sys/* config + [script] + # the script extention of cpi + [ulimit] + # to change the resources limit of user + [schedule_policy] + # to change the schedule policy + [check] + # check the environment + [tip] + # the recommended optimization, which should be performed manunaly ``` ### collection @@ -247,9 +326,9 @@ Collect the global resource usage and OS status information during service runni

Network port used during service running, for example, eth0.

-

--workload_type, -t

+

--app_type, -t

-

Workload type, which is used as a label for training.

+

Mark the application type of the service as a label for training.

--duration, -d

@@ -269,14 +348,14 @@ Collect the global resource usage and OS status information during service runni #### Example ``` -# atune-adm collection --filename name --interval 5 --duration 1200 --output_path /home/data --disk sda --network eth0 --workload_type test_type +# atune-adm collection --filename name --interval 5 --duration 1200 --output_path /home/data --disk sda --network eth0 --app_type test_type ``` ### train #### Function -Use the collected data to train the model. Collect data of at least two workload types during training. Otherwise, an error is reported. +Use the collected data to train the model. Collect data of at least two application types during training. Otherwise, an error is reported. #### Format @@ -304,18 +383,18 @@ Use the CSV file in the **data** directory as the training input. The generate #### Function -Delete a user-defined workload type. +Delete a user-defined profile. #### Format -**atune-adm undefine** +**atune-adm undefine** #### Example -Delete the **test\_type** workload type. +Delete the user-defined profile. ``` -# atune-adm undefine test_type +# atune-adm undefine test_service-test_app-test_scenario ``` ## Querying Profiles @@ -325,26 +404,26 @@ Delete the **test\_type** workload type. #### Function -View the profile content of a workload type. +View the profile content. #### Format -**atune-adm info** _ +**atune-adm info** _ #### Example -View the profile content of webserver. +View the profile content of web-nginx-http-long-connection. ``` -# atune-adm info webserver +# atune-adm info web-nginx-http-long-connection -*** ssl_webserver: +*** web-nginx-http-long-connection: # -# webserver tuned configuration +# nginx http long connection A-Tune configuration # [main] -#TODO CONFIG +include = default-default [kernel_config] #TODO CONFIG @@ -352,11 +431,18 @@ View the profile content of webserver. [bios] #TODO CONFIG +[bootloader.grub2] +iommu.passthrough = 1 + [sysfs] #TODO CONFIG +[systemctl] +sysmonitor = stop +irqbalance = stop + [sysctl] -fs.file-max=6553600 +fs.file-max = 6553600 fs.suid_dumpable = 1 fs.aio-max-nr = 1048576 kernel.shmmax = 68719476736 @@ -384,36 +470,23 @@ net.core.rmem_default = 8388608 net.core.rmem_max = 16777216 net.core.wmem_max = 16777216 -[systemctl] -sysmonitor=stop -irqbalance=stop - -[bootloader.grub2] -selinux=0 -iommu.passthrough=1 - -[tip] -bind your master process to the CPU near the network = affinity -bind your network interrupt to the CPU that has this network = affinity -relogin into the system to enable limits setting = OS - [script] -openssl_hpre = 0 prefetch = off +ethtool = -X {network} hfunc toeplitz [ulimit] {user}.hard.nofile = 102400 {user}.soft.nofile = 102400 -[affinity.task] -#TODO CONFIG - -[affinity.irq] +[schedule_policy] #TODO CONFIG [check] #TODO CONFIG +[tip] +SELinux provides extra control and security features to linux kernel. Disabling SELinux will improve the performance but may cause security risks. = kernel +disable the nginx log = application ```    @@ -429,18 +502,18 @@ You can update the existing profile as required. #### Function -Update an optimization item of a workload type to the content in the **new.conf** file. +Update the original tuning items in the existing profile to the content in the **new.conf** file. #### Format -**atune-adm update** +**atune-adm update** #### Example -Update the workload type to **test\_type** and the optimization item of test\_name to **new.conf**. +Change the tuning item of the profile named **test_service-test_app-test_scenario** to **new.conf**. ``` -# atune-adm update test_type test_name ./new.conf +# atune-adm update test_service-test_app-test_scenario ./new.conf ``` ## Activating a Profile @@ -449,22 +522,22 @@ Update the workload type to **test\_type** and the optimization item of test\_ #### Function -Manually activate a profile of a workload type. +Manually activate the profile to make it in the active state. #### Format -**atune-adm profile **_<_WORKLOAD\_TYPE_\>_ +**atune-adm profile **_<_profile_\>_ #### Parameter Description -You can run the **list** command to query the supported workload types. +For details about the profile name, see the query result of the list command. #### Example -Activate the profile configuration of webserver. +Activate the profile corresponding to the web-nginx-http-long-connection. ``` -# atune-adm profile webserver +# atune-adm profile web-nginx-http-long-connection ``` ## Rolling Back Profiles @@ -561,7 +634,7 @@ Check the CPU, BIOS, OS, and NIC information. ## Automatic Parameter Optimization -A-Tune provides the automatic search capability for optimal configurations, eliminating the need for repeated manual parameter adjustment and performance evaluation. This greatly improves the search efficiency of optimal configurations. +A-Tune provides the automatic search capability with the optimal configuration, saving the trouble of manually configuring parameters and performance evaluation. This greatly improves the search efficiency of optimal configurations. @@ -574,12 +647,12 @@ Use the specified project file to search the dynamic space for parameters and fi #### Format +**atune-adm tuning** \[OPTIONS\] + >![](public_sys-resources/icon-note.gif) **NOTE:** >Before running the command, ensure that the following conditions are met: ->1. The YAML configuration file of the server has been edited and placed in the **/etc/atuned/tuning/** directory on the server by the server administrator. ->2. The YAML configuration file of the client has been edited and placed in an arbitrary directory on the client. - -**atune-adm tuning** \[OPTIONS\] +>1. The YAML configuration file on the server has been edited and stored in the **/etc/atuned/tuning/** directory of the atuned service. +>2. The YAML configuration file of the client has been compiled and stored on the atuned client. #### Parameter Description @@ -601,11 +674,22 @@ Use the specified project file to search the dynamic space for parameters and fi

Specifies the project name in the YAML file to be restored.

+

--restart, -c

+ +

Perform tuning based on historical tuning results.

+ + +

--detail, -d

+ +

Print detailed information about the tuning process.

+ + - + + >![](public_sys-resources/icon-note.gif) **NOTE:** - >The preceding two parameters must be used at the same time, and the -p parameter must be followed by the specific project name. + >If this parameter is used, the -p parameter must be followed by a specific project name and the YAML file of the project must be specified. - **PROJECT\_YAML**: YAML configuration file of the client. @@ -783,15 +867,6 @@ Use the specified project file to search the dynamic space for parameters and fi

The value is user-defined and must be within the valid range of this parameter.

-

ref

- -

Recommended initial value of the parameter

- -

Integer or character string

- -

The value is user-defined and must be within the valid range of this parameter.

- - @@ -817,6 +892,15 @@ Use the specified project file to search the dynamic space for parameters and fi

-

+

engine

+ +

Tuning algorithm.

+ +

Character string

+ +

"random", "forest", "gbrt", "bayes", "extraTrees"

+ +

iterations

Number of optimization iterations.

@@ -826,6 +910,51 @@ Use the specified project file to search the dynamic space for parameters and fi

≥ 10

+

random_starts

+ +

Number of random iterations.

+ +

Integer

+ +

< iterations

+ + +

feature_filter_engine

+ +

Parameter search algorithm.

+ +

Character string

+ +

"lhs"

+ + +

feature_filter_cycle

+ +

Parameter search cycles.

+ +

Integer

+ +

-

+ + +

feature_filter_iters

+ +

Number of iterations for each cycle of parameter search.

+ +

Integer

+ +

-

+ + +

split_count

+ +

Number of evenly selected parameters in the value range of tuning parameters.

+ +

Integer

+ +

-

+ +

benchmark

Performance test script.

@@ -847,6 +976,7 @@ Use the specified project file to search the dynamic space for parameters and fi + **Table 4** Description of evaluations configuration item

Name

@@ -912,86 +1042,36 @@ Use the specified project file to search the dynamic space for parameters and fi The following is an example of the YAML file configuration on a server: ``` -project: "example" -maxiterations: 10 +project: "compress" +maxiterations: 500 startworkload: "" stopworkload: "" object : - - name : "vm.swappiness" + name : "compressLevel" info : - desc : "the vm.swappiness" - get : "sysctl -a | grep vm.swappiness" - set : "sysctl -w vm.swappiness=$value" - needrestart: "false" + desc : "The compresslevel parameter is an integer from 1 to 9 controlling the level of compression" + get : "cat /root/A-Tune/examples/tuning/compress/compress.py | grep 'compressLevel=' | awk -F '=' '{print $2}'" + set : "sed -i 's/compressLevel=\\s*[0-9]*/compressLevel=$value/g' /root/A-Tune/examples/tuning/compress/compress.py" + needrestart : "false" type : "continuous" scope : - - 0 - - 10 - ref : 1 - - - name : "irqbalance" - info : - desc : "system irqbalance" - get : "systemctl status irqbalance" - set : "systemctl $value sysmonitor;systemctl $value irqbalance" - needrestart: "false" - type : "discrete" - options: - - "start" - - "stop" - dtype : "string" - ref : "start" - - - name : "net.tcp_min_tso_segs" - info : - desc : "the minimum tso number" - get : "cat /proc/sys/net/ipv4/tcp_min_tso_segs" - set : "echo $value > /proc/sys/net/ipv4/tcp_min_tso_segs" - needrestart: "false" - type : "continuous" - scope: - 1 - - 16 - ref : 2 + - 9 + dtype : "int" - - name : "prefetcher" + name : "compressMethod" info : - desc : "" - get : "cat /sys/class/misc/prefetch/policy" - set : "echo $value > /sys/class/misc/prefetch/policy" - needrestart: "false" + desc : "The compressMethod parameter is a string controlling the compression method" + get : "cat /root/A-Tune/examples/tuning/compress/compress.py | grep 'compressMethod=' | awk -F '=' '{print $2}' | sed 's/\"//g'" + set : "sed -i 's/compressMethod=\\s*[0-9,a-z,\"]*/compressMethod=\"$value\"/g' /root/A-Tune/examples/tuning/compress/compress.py" + needrestart : "false" type : "discrete" - options: - - "0" - - "15" + options : + - "bz2" + - "zlib" + - "gzip" dtype : "string" - ref : "15" - - - name : "kernel.sched_min_granularity_ns" - info : - desc : "Minimal preemption granularity for CPU-bound tasks" - get : "sysctl kernel.sched_min_granularity_ns" - set : "sysctl -w kernel.sched_min_granularity_ns=$value" - needrestart: "false" - type : "continuous" - scope: - - 5000000 - - 50000000 - ref : 10000000 - - - name : "kernel.sched_latency_ns" - info : - desc : "" - get : "sysctl kernel.sched_latency_ns" - set : "sysctl -w kernel.sched_latency_ns=$value" - needrestart: "false" - type : "continuous" - scope: - - 10000000 - - 100000000 - ref : 16000000 - ```    @@ -999,17 +1079,25 @@ object : The following is an example of the YAML file configuration on a client: ``` -project: "example" -iterations : 10 -benchmark : "sh /home/Benchmarks/mysql/tunning_mysql.sh" +project: "compress" +engine : "gbrt" +iterations : 20 +random_starts : 10 + +benchmark : "python3 /root/A-Tune/examples/tuning/compress/compress.py" evaluations : - - name: "tps" + name: "time" + info: + get: "echo '$out' | grep 'time' | awk '{print $3}'" + type: "positive" + weight: 20 + - + name: "compress_ratio" info: - get: "echo -e '$out' |grep 'transactions:' |awk '{print $3}' | cut -c 2-" + get: "echo '$out' | grep 'compress_ratio' | awk '{print $3}'" type: "negative" - weight: 100 - threshold: 100 + weight: 80 ```    @@ -1019,13 +1107,13 @@ evaluations : - Perform tuning. ``` - # atune-adm tuning example-client.yaml + # atune-adm tuning --project compress --detail compress_client.yaml ``` -- Restore the initial configuration before tuning. The example value is the project name in the YAML file. +- Restore the initial configuration before tuning. The compress is the project name in the YAML file. ``` - # atune-adm tuning --restore --project example + # atune-adm tuning --restore --project compress ``` diff --git a/content/en/docs/A-Tune/getting-to-know-a-tune.md b/content/en/docs/A-Tune/getting-to-know-a-tune.md index 03415ded0cdc5fe96a42f1d872a45dde1ede6c4d..5564c55e8f6a7107f953ae84cb0ebbe0dfbfc18b 100644 --- a/content/en/docs/A-Tune/getting-to-know-a-tune.md +++ b/content/en/docs/A-Tune/getting-to-know-a-tune.md @@ -39,7 +39,6 @@ The following figure shows the A-Tune core technical architecture, which consist **Table 1** Feature maturity - - - @@ -72,18 +71,19 @@ The following figure shows the A-Tune core technical architecture, which consist

Feature

Maturity

@@ -48,14 +47,14 @@ The following figure shows the A-Tune core technical architecture, which consist

Auto optimization of 11 applications in seven workload types

+

Auto optimization of 15 applications in 11 workload types

Tested

Pilot

User-defined workload types and service models

+

User-defined profile and service models

Tested
+ ### Supported Service Models -Based on the workload characteristics of applications, A-Tune classifies services into seven types. For details about the workload characteristics of each type and the applications supported by A-Tune, see [Table 2](#table2819164611311). +Based on the workload characteristics of applications, A-Tune classifies services into 11 types. For details about the bottleneck of each type and the applications supported by A-Tune, see [Table 2](#table2819164611311). **Table 2** Supported workload types and applications -

Workload

+ - @@ -93,87 +93,103 @@ Based on the workload characteristics of applications, A-Tune classifies service - - - - - - - - - - - - - - - - + + + + + - - - - - - - - - - - + + + + +

Service category

Type

Workload Characteristic

+

Bottleneck

Supported Application

Default type

The usage of CPU, memory bandwidth, network, and I/O resources is low.

+

Low resource usage in terms of cpu, memory, network, and I/O

N/A

webserver

HTTPS application

+

Web application

The CPU usage is high.

+

Bottlenecks of cpu and network

Nginx

+

Nginx, Apache Traffic Server

big_database

+

database

Database

  • Relational database

    Read: The usage of CPU, memory bandwidth, and network is high.

    -

    Write: The usage of I/O is high.

    -
-
  • Non-relational database

    The usage of CPU and I/O is high.

    -
+		Bottlenecks of cpu, memory, and I/O

MongoDB, MySQL, PostgreSQL, and MariaDB

+

Mongodb, Mysql, Postgresql, Mariadb

big_data

Big data

The usage of CPU and I/O is high.

+

Bottlenecks of cpu and memory

Hadoop and Spark

+

Hadoop-hdfs, Hadoop-spark

in-memory_computing

+

middleware

Memory-intensive application

+

Middleware framework

The usage of CPU and memory bandwidth is high.

+

Bottlenecks of cpu and network

SPECjbb2015

+

Dubbo

in-memory_database

Computing- and network-intensive application

+

Memory database

The usage of a single-core CPU is high, and the network usage is high in multi-instance scenarios.

+

Bottlenecks of memory and I/O

Redis

single_computer_intensive_jobs

+

basic-test-suite

+

Basic test suite

+

Bottlenecks of cpu and memory

+

SPECCPU2006, SPECjbb2015

+

hpc

Computing-intensive application

+

Human genome

The usage of a single-core CPU is high, and the usage of memory bandwidth of some subitems is high.

+

Bottlenecks of cpu, memory, and I/O

SPECCPU2006

+

Gatk4

communication

+

storage

Network-intensive application

+

Storage

The usage of CPU and network is high.

+

Bottlenecks of network, and I/O

Dubbo

+

Ceph

idle

+

virtualization

System in idle state

+

Virtualization

The system is in idle state and no applications are running.

+

Bottlenecks of cpu, memory, and I/O

N/A

+

Consumer-cloud, Mariadb

+

docker

+

Docker

+

Bottlenecks of cpu, memory, and I/O

+

Mariadb
+ + diff --git a/content/en/docs/A-Tune/installation-and-deployment.md b/content/en/docs/A-Tune/installation-and-deployment.md index 96a1e025d01cdf9261809680afb1e6e42115feaf..f4c9fb46ae7e0f6912135b1ff08bfc444099b32b 100644 --- a/content/en/docs/A-Tune/installation-and-deployment.md +++ b/content/en/docs/A-Tune/installation-and-deployment.md @@ -10,6 +10,7 @@ This chapter describes how to install and deploy A-Tune. - [Installation Procedure](#installation-procedure) - [A-Tune Deployment](#a-tune-deployment) - [Starting A-Tune](#starting-a-tune) + - [Starting A-Tune engine](#starting-a-tune-engine) @@ -84,16 +85,17 @@ To install the A-Tune, perform the following steps: ``` -4. Install an A-Tune server. +4. Install an A-Tune server. >![](public_sys-resources/icon-note.gif) **NOTE:** >In this step, both the server and client software packages are installed. For the single-node deployment, skip **Step 5**. ``` # yum install atune -y + # yum install atune-engine -y ``` -5. For a distributed mode, install an A-Tune client. +5. For a distributed mode, install an A-Tune client on associated server. ``` # yum install atune-client -y @@ -106,8 +108,9 @@ To install the A-Tune, perform the following steps: atune-client-xxx atune-db-xxx atune-xxx - ``` - + atune-engine-xxx +``` + If the preceding information is displayed, the installation is successful. @@ -121,16 +124,37 @@ This chapter describes how to deploy A-Tune. The configuration items in the A-Tune configuration file **/etc/atuned/atuned.cnf** are described as follows: -- A-Tune service startup configuration +- A-Tune service startup configuration You can modify the parameter value as required. - **protocol**: Protocol used by the gRPC service. The value can be **unix** or **tcp**. **unix** indicates the local socket communication mode, and **tcp** indicates the socket listening port mode. The default value is **unix**. - - **address**: Listening IP address of the gRPC service. The default value is **unix socket**. If the gRPC service is deployed in distributed mode, change the value to the listening IP address. - - **port**: Listening port of the gRPC server. The value ranges from 0 to 65535. If **protocol** is set to **unix**, you do not need to set this parameter. - - **rest\_port**: Listening port of the system REST service. The value ranges from 0 to 65535. - - **sample\_num**: Number of samples collected when the system executes the analysis process. + - **port**: Listening port of the gRPC server. The value ranges from 0 to 65535. If **protocol** is set to **unix**, you do not need to set this parameter. + - **connect**: IP address list of the nodes where the A-Tune is located when the A-Tune is deployed in a cluster. IP addresses are separated by commas (,). + - **rest_host**: Listening address of the REST service. The default value is localhost. + - **rest_port**: Listening port of the REST service. The value ranges from 0 to 65535. The default value is 8383. + - **engine_host**: IP address for connecting to the A-Tune engine service of the system. + - **engine_port**: Port for connecting to the A-Tune engine service of the system. + - **sample_num**: Number of samples collected when the system executes the analysis process. The default value is 20. + - **interval**: Interval for collecting samples when the system executes the analysis process. The default value is 5s. + - **grpc_tls**: Indicates whether to enable SSL/TLS certificate verification for the gRPC service. By default, this function is disabled. After grpc_tls is enabled, you need to set the following environment variables before running the **atune-adm** command to communicate with the server: + - export ATUNE_TLS=yes + - export ATUNED_CACERT= + - export ATUNED_CLIENTCERT= + - export ATUNED_CLIENTKEY= + - export ATUNED_SERVERCN=server + - **tlsservercafile**: Path of the gPRC server's CA certificate. + - **tlsservercertfile**: Path of the gPRC server certificate. + - **tlsserverkeyfile**: Path of the gPRC server key. + - **rest_tls**: Indicates whether to enable SSL/TLS certificate verification for the REST service. This function is enabled by default. + - **tlsrestcacertfile**: Path of the server's CA certificate of the REST service. + - **tlsrestservercertfile**: Path of the server certificate of the REST service. + - **tlsrestserverkeyfile**: Indicates the key path of the REST service. + - **engine_tls**: Indicates whether to enable SSL/TLS certificate verification for the A-Tune engine service. This function is enabled by default.. + - **tlsenginecacertfile**: Path of the client CA certificate of the A-Tune engine service. + - **tlsengineclientcertfile**: Client certificate path of the A-Tune engine service. + - **tlsengineclientkeyfile**: Client key path of the A-Tune engine service. - System information @@ -139,104 +163,201 @@ The configuration items in the A-Tune configuration file **/etc/atuned/atuned.c - **disk**: Disk information to be collected during the analysis process or specified disk during disk optimization. - **network**: NIC information to be collected during the analysis process or specified NIC during NIC optimization. - **user**: User name used for ulimit optimization. Currently, only the user **root** is supported. - - **tls**: SSL/TLS certificate verification for the gRPC and HTTP services of A-Tune. This is disabled by default. After TLS is enabled, you need to set the following environment variables before running the **atune-adm** command to communicate with the server: - - export ATUNE\_TLS=yes - - export ATUNE\_CLICERT= - - - **tlsservercertfile**: path of the gPRC server certificate. - - **tlsserverkeyfile**: gPRC server key path. - - **tlshttpcertfile**: HTTP server certificate path. - - **tlshttpkeyfile**: HTTP server key path. - - **tlshttpcacertfile**: CA certificate path of the HTTP server. - + - Log information - Change the log path and level based on the site requirements. By default, the log information is stored in **/var/log/messages**. + Change the log level as required. The default log level is info. Log information is recorded in the **/var/log/messages** file. - Monitor information Hardware information that is collected by default when the system is started. + +- Tuning information + + Tuning is the parameter information required for offline tuning. + + - **noise**: Evaluation value of Gaussian noise. + - **sel_feature**: Indicates whether to enable the function of generating the importance ranking of offline tuning parameters. By default, this function is disabled. + + +#### Example + +``` +#################################### server ############################### + # atuned config + [server] + # the protocol grpc server running on + # ranges: unix or tcp + protocol = unix + + # the address that the grpc server to bind to + # default is unix socket /var/run/atuned/atuned.sock + # ranges: /var/run/atuned/atuned.sock or ip address + address = /var/run/atuned/atuned.sock + + # the atune nodes in cluster mode, separated by commas + # it is valid when protocol is tcp + # connect = ip01,ip02,ip03 + + # the atuned grpc listening port + # the port can be set between 0 to 65535 which not be used + # port = 60001 + + # the rest service listening port, default is 8383 + # the port can be set between 0 to 65535 which not be used + rest_host = localhost + rest_port = 8383 + + # the tuning optimizer host and port, start by engine.service + # if engine_host is same as rest_host, two ports cannot be same + # the port can be set between 0 to 65535 which not be used + engine_host = localhost + engine_port = 3838 + + # when run analysis command, the numbers of collected data. + # default is 20 + sample_num = 20 + + # interval for collecting data, default is 5s + interval = 5 + + # enable gRPC authentication SSL/TLS + # default is false + # grpc_tls = false + # tlsservercafile = /etc/atuned/grpc_certs/ca.crt + # tlsservercertfile = /etc/atuned/grpc_certs/server.crt + # tlsserverkeyfile = /etc/atuned/grpc_certs/server.key + + # enable rest server authentication SSL/TLS + # default is true + rest_tls = true + tlsrestcacertfile = /etc/atuned/rest_certs/ca.crt + tlsrestservercertfile = /etc/atuned/rest_certs/server.crt + tlsrestserverkeyfile = /etc/atuned/rest_certs/server.key + + # enable engine server authentication SSL/TLS + # default is true + engine_tls = true + tlsenginecacertfile = /etc/atuned/engine_certs/ca.crt + tlsengineclientcertfile = /etc/atuned/engine_certs/client.crt + tlsengineclientkeyfile = /etc/atuned/engine_certs/client.key + + + #################################### log ############################### + [log] + # either "debug", "info", "warn", "error", "critical", default is "info" + level = info + + #################################### monitor ############################### + [monitor] + # with the module and format of the MPI, the format is {module}_{purpose} + # the module is Either "mem", "net", "cpu", "storage" + # the purpose is "topo" + module = mem_topo, cpu_topo + + #################################### system ############################### + # you can add arbitrary key-value here, just like key = value + # you can use the key in the profile + [system] + # the disk to be analysis + disk = sda + + # the network to be analysis + network = enp189s0f0 + + user = root + + #################################### tuning ############################### + # tuning configs + [tuning] + noise = 0.000000001 + sel_feature = false +``` + +The configuration items in the configuration file **/etc/atuned/engine.cnf** of the A-Tune engine are described as follows: + +- Startup configuration of the A-Tune engine service + + You can modify the startup configuration as required. + + - **engine_host**: Listening address of the A-Tune engine service. The default value is localhost. + - **engine_port**: Listening port of the A-Tune engine service. The value ranges from 0 to 65535. The default value is 3838. + - **engine_tls**: Indicates whether to enable SSL/TLS certificate verification for the A-Tune engine service. This function is enabled by default. + - **tlsenginecacertfile**: Path of the server CA certificate of the A-Tune engine service. + - **tlsengineservercertfile**: Path of the server certificate of the A-Tune engine service. + - **tlsengineserverkeyfile**: Server key path of the A-Tune engine service. + +- Log information + + Change the log level as required. The default log level is info. Log information is recorded in the **/var/log/messages** file. #### Example ``` -#################################### server ############################### -# atuned config -[server] -# the protocol grpc server running on -# ranges: unix or tcp -protocol = unix - -# the address that the grpc server to bind to -# default is unix socket /var/run/atuned/atuned.sock -# ranges: /var/run/atuned/atuned.sock or ip -address = /var/run/atuned/atuned.sock - -# the atuned grpc listening port, default is 60001 -# the port can be set between 0 to 65535 which not be used -port = 60001 - -# the rest service listening port, default is 8383 -# the port can be set between 0 to 65535 which not be used -rest_port = 8383 - -# when run analysis command, the numbers of collected data. -# default is 20 -sample_num = 20 - -# Enable gRPC and http server authentication SSL/TLS -# default is false -# tls = true -# tlsservercertfile = /etc/atuned/server.pem -# tlsserverkeyfile = /etc/atuned/server.key -# tlshttpcertfile = /etc/atuned/http/server.pem -# tlshttpkeyfile = /etc/atuned/http/server.key -# tlshttpcacertfile = /etc/atuned/http/cacert.pem - -#################################### log ############################### -# Either "debug", "info", "warn", "error", "critical", default is "info" -level = info - -#################################### monitor ############################### -[monitor] -# With the module and format of the MPI, the format is {module}_{purpose} -# The module is Either "mem", "net", "cpu", "storage" -# The purpose is "topo" -module = mem_topo, cpu_topo - -#################################### system ############################### -# you can add arbitrary key-value here, just like key = value -# you can use the key in the profile -[system] -# the disk to be analysis -disk = sda - -# the network to be analysis -network = enp189s0f0 - -user = root +#################################### engine ############################### + [server] + # the tuning optimizer host and port, start by engine.service + # if engine_host is same as rest_host, two ports cannot be same + # the port can be set between 0 to 65535 which not be used + engine_host = localhost + engine_port = 3838 + + # enable engine server authentication SSL/TLS + # default is true + engine_tls = true + tlsenginecacertfile = /etc/atuned/engine_certs/ca.crt + tlsengineservercertfile = /etc/atuned/engine_certs/server.crt + tlsengineserverkeyfile = /etc/atuned/engine_certs/server.key + + #################################### log ############################### + [log] + # either "debug", "info", "warn", "error", "critical", default is "info" + level = info ``` ## Starting A-Tune After the A-Tune is installed, you need to start the A-Tune service. -- Start the atuned service. +- Start the atuned service. + + ``` + # systemctl start atuned + ``` + + +- Query the atuned service status. + + ``` + # systemctl status atuned + ``` + + If the following command output is displayed, the service is started successfully: + + ![](figures/en-us_image_0214540398.png) + +## Starting A-Tune engine + +To use AI functions, you need to start the A-Tune engine service. + +- Start the atune-engine service. ``` - # systemctl start atuned + # systemctl start atune-engine ``` -- To query the status of the atuned service, run the following command: +- Query the atune-engine service status. ``` - # systemctl status atuned + # systemctl status atune-engine ``` - If the following information is displayed, the service is started successfully: + If the following command output is displayed, the service is started successfully: - ![](figures/en-us_image_0214540398.png) + ![](figures/en-us_image_0245342444.png) + + diff --git "a/content/zh/docs/A-Tune/\345\256\211\350\243\205\344\270\216\351\203\250\347\275\262.md" "b/content/zh/docs/A-Tune/\345\256\211\350\243\205\344\270\216\351\203\250\347\275\262.md" index f6036329e49b53edfe6f048887491914b3b52170..4fec23f0ba7f6455898e41442bc7dd6a3941ff9a 100644 --- "a/content/zh/docs/A-Tune/\345\256\211\350\243\205\344\270\216\351\203\250\347\275\262.md" +++ "b/content/zh/docs/A-Tune/\345\256\211\350\243\205\344\270\216\351\203\250\347\275\262.md" @@ -12,8 +12,8 @@ - [部署A-Tune](#部署A-Tune) - [配置介绍](#配置介绍) - [启动A-Tune](#启动A-Tune) -- [启动A-Tune engine](#启动A-Tune engine) - + - [启动A-Tune engine](#启动A-Tune engine) + ## 软硬件要求