From ab2dc3c8c537422b7736335f54d0d657649a8c57 Mon Sep 17 00:00:00 2001 From: ylzhangah <1194926515@qq.com> Date: Thu, 9 Oct 2025 17:31:59 +0800 Subject: [PATCH] Deployment script internationalization --- deploy/en/chart/README.md | 10 + deploy/en/chart/agents/.helmignore | 23 + deploy/en/chart/agents/Chart.yaml | 6 + .../agents/configs/ai-infra-agent/.env.yaml | 3 + .../agents/configs/rca-agent/config.json | 343 +++++++++++++ .../configs/rca-agent/request_config.json | 4 + deploy/en/chart/agents/configs/tune/.env.yaml | 15 + .../chart/agents/configs/tune/app_config.yaml | 3 + .../agents/configs/tune/knob_rag_config.json | 4 + .../agents/configs/tune/optimize_config.yaml | 3 + deploy/en/chart/agents/templates/NOTES.txt | 1 + .../ai-infra-agent/ai-infra-deployment.yaml | 55 ++ .../ai-infra-agent/ai-infra-secret.yaml | 11 + .../ai-infra-agent/ai-infra-service.yaml | 17 + .../templates/rca-agent/rca-agent-config.yaml | 12 + .../rca-agent/rca-agent-deployment.yaml | 51 ++ .../rca-agent/rca-agent-service.yaml | 17 + .../templates/tune/tune-deployment.yaml | 52 ++ .../agents/templates/tune/tune-secret.yaml | 17 + .../agents/templates/tune/tune-service.yaml | 17 + deploy/en/chart/agents/values.yaml | 100 ++++ deploy/en/chart/authhub/.helmignore | 23 + deploy/en/chart/authhub/Chart.yaml | 6 + .../authhub/configs/backend/aops-config.yml | 30 ++ .../chart/authhub/configs/backend/authhub.yml | 4 + .../en/chart/authhub/configs/mysql/init.sql | 112 ++++ deploy/en/chart/authhub/templates/NOTES.txt | 7 + .../backend/authhub-backend-config.yaml | 32 ++ .../templates/backend/authhub-backend.yaml | 101 ++++ .../authhub/templates/mysql/mysql-config.yaml | 10 + .../templates/mysql/mysql-storage.yaml | 16 + .../chart/authhub/templates/mysql/mysql.yaml | 90 ++++ .../en/chart/authhub/templates/secrets.yaml | 24 + .../templates/web/authhub-web-config.yaml | 44 ++ .../authhub/templates/web/authhub-web.yaml | 67 +++ deploy/en/chart/authhub/values.yaml | 72 +++ deploy/en/chart/databases/.helmignore | 23 + deploy/en/chart/databases/Chart.yaml | 6 + deploy/en/chart/databases/templates/NOTES.txt | 3 + .../en/chart/databases/templates/_helpers.tpl | 15 + .../templates/minio/minio-storage.yaml | 16 + .../databases/templates/minio/minio.yaml | 79 +++ .../templates/mongo/mongo-config.yaml | 23 + .../templates/mongo/mongo-storage.yaml | 16 + .../databases/templates/mongo/mongo.yaml | 102 ++++ .../templates/opengauss/opengauss-config.yaml | 16 + .../opengauss/opengauss-storage.yaml | 17 + .../templates/opengauss/opengauss.yaml | 93 ++++ .../templates/pgsql/pgsql-config.yaml | 13 + .../templates/pgsql/pgsql-storage.yaml | 16 + .../databases/templates/pgsql/pgsql.yaml | 84 +++ .../databases/templates/redis/redis.yaml | 77 +++ .../en/chart/databases/templates/secrets.yaml | 30 ++ deploy/en/chart/databases/values.yaml | 106 ++++ deploy/en/chart/euler_copilot/.helmignore | 23 + deploy/en/chart/euler_copilot/Chart.yaml | 6 + .../configs/framework/config.toml | 66 +++ .../en/chart/euler_copilot/configs/rag/.env | 60 +++ .../chart/euler_copilot/configs/rag/.env-sql | 29 ++ .../chart/euler_copilot/templates/NOTES.txt | 5 + .../templates/framework/framework-config.yaml | 21 + .../framework/framework-storage.yaml | 31 ++ .../templates/framework/framework.yaml | 120 +++++ .../templates/rag-web/rag-web-config.yaml | 10 + .../templates/rag-web/rag-web.yaml | 80 +++ .../templates/rag/rag-config.yaml | 32 ++ .../euler_copilot/templates/rag/rag.yaml | 112 ++++ .../euler_copilot/templates/secrets.yaml | 36 ++ .../templates/web/web-config.yaml | 11 + .../templates/web/web-storage.yaml | 15 + .../euler_copilot/templates/web/web.yaml | 87 ++++ deploy/en/chart/euler_copilot/values.yaml | 155 ++++++ .../rca/rca-anteater/anteater-deployment.yaml | 49 ++ .../rca/rca-anteater/anteater-secret.yaml | 14 + deploy/en/chart_ssl/traefik-config.yml | 15 + deploy/en/chart_ssl/traefik-secret.yaml | 11 + deploy/en/chart_ssl/traefik-tlsstore.yaml | 9 + .../0-one-click-deploy/one-click-deploy.sh | 294 +++++++++++ deploy/en/scripts/1-check-env/check_env.sh | 250 +++++++++ .../scripts/2-install-tools/install_tools.sh | 381 ++++++++++++++ .../3-install-ollama/install_ollama.sh | 337 ++++++++++++ .../4-deploy-deepseek/deploy_deepseek.sh | 252 +++++++++ .../5-deploy-embedding/deploy-embedding.sh | 252 +++++++++ .../6-install-databases/install_databases.sh | 169 ++++++ .../7-install-authhub/install_authhub.sh | 240 +++++++++ .../install_eulercopilot.sh | 480 ++++++++++++++++++ .../9-other-script/backup_databases.sh | 289 +++++++++++ .../scripts/9-other-script/download_file.sh | 49 ++ .../get_client_id_and_secret.py | 172 +++++++ deploy/en/scripts/9-other-script/get_log.sh | 37 ++ .../scripts/9-other-script/import_images.sh | 258 ++++++++++ .../install_oidc_eulercopilot.sh | 268 ++++++++++ .../9-other-script/migrate_minio_database.sh | 244 +++++++++ .../9-other-script/migrate_mysql_database.sh | 262 ++++++++++ .../migrate_opengauss_database.sh | 210 ++++++++ .../modify_eulercopilot_yaml.py | 120 +++++ .../scripts/9-other-script/prepare_docker.sh | 163 ++++++ .../en/scripts/9-other-script/save_images.sh | 187 +++++++ deploy/en/scripts/deploy.sh | 329 ++++++++++++ deploy/en/secret_helper/Dockerfile | 14 + deploy/en/secret_helper/__init__.py | 0 deploy/en/secret_helper/config.example.yaml | 14 + deploy/en/secret_helper/file_copy.py | 86 ++++ deploy/en/secret_helper/job.py | 11 + deploy/en/secret_helper/main.py | 41 ++ deploy/en/secret_helper/requirements.txt | 1 + 106 files changed, 8544 insertions(+) create mode 100644 deploy/en/chart/README.md create mode 100644 deploy/en/chart/agents/.helmignore create mode 100644 deploy/en/chart/agents/Chart.yaml create mode 100644 deploy/en/chart/agents/configs/ai-infra-agent/.env.yaml create mode 100644 deploy/en/chart/agents/configs/rca-agent/config.json create mode 100644 deploy/en/chart/agents/configs/rca-agent/request_config.json create mode 100644 deploy/en/chart/agents/configs/tune/.env.yaml create mode 100644 deploy/en/chart/agents/configs/tune/app_config.yaml create mode 100644 deploy/en/chart/agents/configs/tune/knob_rag_config.json create mode 100644 deploy/en/chart/agents/configs/tune/optimize_config.yaml create mode 100644 deploy/en/chart/agents/templates/NOTES.txt create mode 100644 deploy/en/chart/agents/templates/ai-infra-agent/ai-infra-deployment.yaml create mode 100644 deploy/en/chart/agents/templates/ai-infra-agent/ai-infra-secret.yaml create mode 100644 deploy/en/chart/agents/templates/ai-infra-agent/ai-infra-service.yaml create mode 100644 deploy/en/chart/agents/templates/rca-agent/rca-agent-config.yaml create mode 100644 deploy/en/chart/agents/templates/rca-agent/rca-agent-deployment.yaml create mode 100644 deploy/en/chart/agents/templates/rca-agent/rca-agent-service.yaml create mode 100644 deploy/en/chart/agents/templates/tune/tune-deployment.yaml create mode 100644 deploy/en/chart/agents/templates/tune/tune-secret.yaml create mode 100644 deploy/en/chart/agents/templates/tune/tune-service.yaml create mode 100644 deploy/en/chart/agents/values.yaml create mode 100644 deploy/en/chart/authhub/.helmignore create mode 100644 deploy/en/chart/authhub/Chart.yaml create mode 100644 deploy/en/chart/authhub/configs/backend/aops-config.yml create mode 100644 deploy/en/chart/authhub/configs/backend/authhub.yml create mode 100644 deploy/en/chart/authhub/configs/mysql/init.sql create mode 100644 deploy/en/chart/authhub/templates/NOTES.txt create mode 100644 deploy/en/chart/authhub/templates/backend/authhub-backend-config.yaml create mode 100644 deploy/en/chart/authhub/templates/backend/authhub-backend.yaml create mode 100644 deploy/en/chart/authhub/templates/mysql/mysql-config.yaml create mode 100644 deploy/en/chart/authhub/templates/mysql/mysql-storage.yaml create mode 100644 deploy/en/chart/authhub/templates/mysql/mysql.yaml create mode 100644 deploy/en/chart/authhub/templates/secrets.yaml create mode 100644 deploy/en/chart/authhub/templates/web/authhub-web-config.yaml create mode 100644 deploy/en/chart/authhub/templates/web/authhub-web.yaml create mode 100644 deploy/en/chart/authhub/values.yaml create mode 100644 deploy/en/chart/databases/.helmignore create mode 100644 deploy/en/chart/databases/Chart.yaml create mode 100644 deploy/en/chart/databases/templates/NOTES.txt create mode 100644 deploy/en/chart/databases/templates/_helpers.tpl create mode 100644 deploy/en/chart/databases/templates/minio/minio-storage.yaml create mode 100644 deploy/en/chart/databases/templates/minio/minio.yaml create mode 100644 deploy/en/chart/databases/templates/mongo/mongo-config.yaml create mode 100644 deploy/en/chart/databases/templates/mongo/mongo-storage.yaml create mode 100644 deploy/en/chart/databases/templates/mongo/mongo.yaml create mode 100644 deploy/en/chart/databases/templates/opengauss/opengauss-config.yaml create mode 100644 deploy/en/chart/databases/templates/opengauss/opengauss-storage.yaml create mode 100644 deploy/en/chart/databases/templates/opengauss/opengauss.yaml create mode 100644 deploy/en/chart/databases/templates/pgsql/pgsql-config.yaml create mode 100644 deploy/en/chart/databases/templates/pgsql/pgsql-storage.yaml create mode 100644 deploy/en/chart/databases/templates/pgsql/pgsql.yaml create mode 100644 deploy/en/chart/databases/templates/redis/redis.yaml create mode 100644 deploy/en/chart/databases/templates/secrets.yaml create mode 100644 deploy/en/chart/databases/values.yaml create mode 100644 deploy/en/chart/euler_copilot/.helmignore create mode 100644 deploy/en/chart/euler_copilot/Chart.yaml create mode 100644 deploy/en/chart/euler_copilot/configs/framework/config.toml create mode 100644 deploy/en/chart/euler_copilot/configs/rag/.env create mode 100644 deploy/en/chart/euler_copilot/configs/rag/.env-sql create mode 100644 deploy/en/chart/euler_copilot/templates/NOTES.txt create mode 100644 deploy/en/chart/euler_copilot/templates/framework/framework-config.yaml create mode 100644 deploy/en/chart/euler_copilot/templates/framework/framework-storage.yaml create mode 100644 deploy/en/chart/euler_copilot/templates/framework/framework.yaml create mode 100644 deploy/en/chart/euler_copilot/templates/rag-web/rag-web-config.yaml create mode 100644 deploy/en/chart/euler_copilot/templates/rag-web/rag-web.yaml create mode 100644 deploy/en/chart/euler_copilot/templates/rag/rag-config.yaml create mode 100644 deploy/en/chart/euler_copilot/templates/rag/rag.yaml create mode 100644 deploy/en/chart/euler_copilot/templates/secrets.yaml create mode 100644 deploy/en/chart/euler_copilot/templates/web/web-config.yaml create mode 100644 deploy/en/chart/euler_copilot/templates/web/web-storage.yaml create mode 100644 deploy/en/chart/euler_copilot/templates/web/web.yaml create mode 100644 deploy/en/chart/euler_copilot/values.yaml create mode 100644 deploy/en/chart/rca/template/rca/rca-anteater/anteater-deployment.yaml create mode 100644 deploy/en/chart/rca/template/rca/rca-anteater/anteater-secret.yaml create mode 100644 deploy/en/chart_ssl/traefik-config.yml create mode 100644 deploy/en/chart_ssl/traefik-secret.yaml create mode 100644 deploy/en/chart_ssl/traefik-tlsstore.yaml create mode 100755 deploy/en/scripts/0-one-click-deploy/one-click-deploy.sh create mode 100755 deploy/en/scripts/1-check-env/check_env.sh create mode 100755 deploy/en/scripts/2-install-tools/install_tools.sh create mode 100755 deploy/en/scripts/3-install-ollama/install_ollama.sh create mode 100755 deploy/en/scripts/4-deploy-deepseek/deploy_deepseek.sh create mode 100755 deploy/en/scripts/5-deploy-embedding/deploy-embedding.sh create mode 100755 deploy/en/scripts/6-install-databases/install_databases.sh create mode 100755 deploy/en/scripts/7-install-authhub/install_authhub.sh create mode 100755 deploy/en/scripts/8-install-EulerCopilot/install_eulercopilot.sh create mode 100755 deploy/en/scripts/9-other-script/backup_databases.sh create mode 100755 deploy/en/scripts/9-other-script/download_file.sh create mode 100755 deploy/en/scripts/9-other-script/get_client_id_and_secret.py create mode 100755 deploy/en/scripts/9-other-script/get_log.sh create mode 100755 deploy/en/scripts/9-other-script/import_images.sh create mode 100755 deploy/en/scripts/9-other-script/install_oidc_eulercopilot.sh create mode 100755 deploy/en/scripts/9-other-script/migrate_minio_database.sh create mode 100755 deploy/en/scripts/9-other-script/migrate_mysql_database.sh create mode 100755 deploy/en/scripts/9-other-script/migrate_opengauss_database.sh create mode 100755 deploy/en/scripts/9-other-script/modify_eulercopilot_yaml.py create mode 100755 deploy/en/scripts/9-other-script/prepare_docker.sh create mode 100755 deploy/en/scripts/9-other-script/save_images.sh create mode 100755 deploy/en/scripts/deploy.sh create mode 100644 deploy/en/secret_helper/Dockerfile create mode 100644 deploy/en/secret_helper/__init__.py create mode 100644 deploy/en/secret_helper/config.example.yaml create mode 100644 deploy/en/secret_helper/file_copy.py create mode 100644 deploy/en/secret_helper/job.py create mode 100644 deploy/en/secret_helper/main.py create mode 100644 deploy/en/secret_helper/requirements.txt diff --git a/deploy/en/chart/README.md b/deploy/en/chart/README.md new file mode 100644 index 000000000..f0b9c6b87 --- /dev/null +++ b/deploy/en/chart/README.md @@ -0,0 +1,10 @@ +# EulerCopilot + +## Deployment Sequence + +1. databases [Required] +2. authhub [Required] +3. witchaind [Required] +4. euler-copilot [Required] +5. rca +6. agents diff --git a/deploy/en/chart/agents/.helmignore b/deploy/en/chart/agents/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/deploy/en/chart/agents/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/deploy/en/chart/agents/Chart.yaml b/deploy/en/chart/agents/Chart.yaml new file mode 100644 index 000000000..d6d275524 --- /dev/null +++ b/deploy/en/chart/agents/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: euler-copilot-helm +description: Euler Copilot Helm Chart +type: application +version: 0.9.1 +appVersion: "1.16.0" diff --git a/deploy/en/chart/agents/configs/ai-infra-agent/.env.yaml b/deploy/en/chart/agents/configs/ai-infra-agent/.env.yaml new file mode 100644 index 000000000..26e477997 --- /dev/null +++ b/deploy/en/chart/agents/configs/ai-infra-agent/.env.yaml @@ -0,0 +1,3 @@ +# Fastapi +UVICORN_IP: 0.0.0.0 +UVICORN_PORT: 8101 \ No newline at end of file diff --git a/deploy/en/chart/agents/configs/rca-agent/config.json b/deploy/en/chart/agents/configs/rca-agent/config.json new file mode 100644 index 000000000..f7df7983c --- /dev/null +++ b/deploy/en/chart/agents/configs/rca-agent/config.json @@ -0,0 +1,343 @@ +{ + "profiling_config": { + "anomaly_mapping": { + "gala_gopher_container_cpu_user_seconds_total": [ + "oncpu_sample", + "oncpu" + ], + "gala_gopher_sli_container_cpu_rundelay": [ + "oncpu_sample", + "oncpu" + ], + "gala_gopher_container_cpu_system_seconds_total": [ + "oncpu_sample", + "oncpu" + ], + "gala_gopher_container_memory_working_set_bytes": [ + "mem_usage" + ], + "gala_gopher_container_memory_rss": [ + "mem_usage" + ], + "gala_gopher_container_memory_cache": [ + "mem_usage" + ], + "gala_gopher_container_memory_mapped_file": [ + "mem_usage" + ], + "gala_gopher_container_fs_reads_bytes_total": [ + "syscall_file" + ], + "gala_gopher_container_fs_read_seconds_total": [ + "syscall_file" + ], + "gala_gopher_container_fs_writes_bytes_total": [ + "syscall_file" + ], + "gala_gopher_container_fs_write_seconds_total": [ + "syscall_file" + ] + }, + "profiling_duration_mapping": { + "gala_gopher_container_cpu_user_seconds_total": 45, + "gala_gopher_sli_container_cpu_rundelay": 45, + "gala_gopher_container_cpu_system_seconds_total": 45, + "gala_gopher_container_memory_working_set_bytes": 45, + "gala_gopher_container_memory_rss": 45, + "gala_gopher_container_memory_cache": 45, + "gala_gopher_container_memory_mapped_file": 45, + "gala_gopher_container_fs_reads_bytes_total": 45, + "gala_gopher_container_fs_read_seconds_total": 45, + "gala_gopher_container_fs_writes_bytes_total": 45, + "gala_gopher_container_fs_write_seconds_total": 45 + }, + "gopher_container_id": "5e54f72e9921", + "remote_host": "192.168.10.33", + "remote_password": "bNM2RN/PvcLCDg==" + }, + "kafka": { + "server": "kafka.gala.svc.cluster.local", + "port": "9092", + "storage_topic": "usad_intermediate_results", + "anteater_result_topic": "gala_anteater_hybrid_model", + "rca_result_topic": "gala_cause_inference_test", + "meta_topic": "gala_gopher_metadata" + }, + "prometheus": { + "server": "kube-prometheus-stack-1711-prometheus.monitoring.svc.cluster.local", + "port": "9090", + "steps": 5 + }, + "elastic_search": { + "es_url": "http://10.137.17.123:9200/" + }, + "meta_graph": { + "a7d32826-2566-40f0-8cd4-feb1a925f4c0": [ + "9a571dfa-8fa3-4b08-ab8f-e86357e1c135", + "ebfde89e-5679-42c2-bc13-39f2a7c6a952" + ], + "ebfde89e-5679-42c2-bc13-39f2a7c6a952": [ + "9a571dfa-8fa3-4b08-ab8f-e86357e1c135", + "63487c96-d466-4fd7-9579-7599a73c222a", + "a7d32826-2566-40f0-8cd4-feb1a925f4c0" + ], + "63487c96-d466-4fd7-9579-7599a73c222a": [ + "9a571dfa-8fa3-4b08-ab8f-e86357e1c135", + "ebfde89e-5679-42c2-bc13-39f2a7c6a952" + ], + "6ed6ac36-3469-4ae8-a859-10256deda463": [ + "a7d32826-2566-40f0-8cd4-feb1a925f4c0", + "ebfde89e-5679-42c2-bc13-39f2a7c6a952", + "63487c96-d466-4fd7-9579-7599a73c222a" + ], + "7f47acbd-33b3-4b0c-a9bd-2d6b456d1b5b": [ + "a7d32826-2566-40f0-8cd4-feb1a925f4c0", + "ebfde89e-5679-42c2-bc13-39f2a7c6a952", + "63487c96-d466-4fd7-9579-7599a73c222a" + ], + "400024f2-15d6-4336-a725-117340f07e71": [ + "6ed6ac36-3469-4ae8-a859-10256deda463", + "7f47acbd-33b3-4b0c-a9bd-2d6b456d1b5b" + ], + "2489c586-b9d2-4759-b433-a6b4f7639389": [ + "400024f2-15d6-4336-a725-117340f07e71" + ], + "69712ca8-8e96-4a41-a84b-c693d8d79555": [ + "400024f2-15d6-4336-a725-117340f07e71" + ] + }, + "cause_root": [ + "nic", + "dns", + "container", + "sli", + "l7", + "cpu", + "disk", + "block", + "bulk", + "mem", + "tcp", + "endpoint", + "proc" + ], + "topology": { + "29a54915-7c9f-479a-9ec7-c1e2de78daf1": { + "front_end_metric": "gala_gopher_sli_rtt_nsec", + "type": "pod", + "point_to": [ + "0d25f2f2-b19b-4b24-b12e-94091ce4009c-192.168.122.18" + ] + }, + "02fa4f55-72bd-4c94-ba92-961bef614b63": { + "front_end_metric": "gala_gopher_sli_tps", + "type": "pod", + "point_to": [ + "0d25f2f2-b19b-4b24-b12e-94091ce4009c-192.168.122.18" + ] + }, + "3031c552-e6b0-43cf-9b2c-770d9c7fdfde": { + "front_end_metric": "gala_gopher_l7_latency_avg", + "type": "pod", + "point_to": [ + "9bd76d8e-25d7-43d1-adba-5230c5f7618c-192.168.122.17" + ] + }, + "e375c612-22a6-401e-928e-f076b56cdd0e": { + "front_end_metric": "gala_gopher_l7_latency_avg", + "type": "pod", + "point_to": [ + "1ea59cbc-f1e8-41a5-8684-eb04451cfefc-192.168.122.19" + ] + } + }, + "special_sli_metrics": [ + "gala_gopher_sli_rtt_nsec", + "gala_gopher_sli_tps", + "gala_gopher_l7_latency_avg", + "gala_gopher_l7_latency_sum", + "gala_gopher_l7_throughput_req", + "gala_gopher_l7_throughput_resp", + "gala_gopher_l7_err_ratio", + "gala_gopher_l7_err_count", + "gala_gopher_proc_flow_perf_tx_delay", + "gala_gopher_proc_flow_perf_rx_delay" + ], + "host_proc_mapping": { + "cpu": [ + "gala_gopher_proc_utime_jiffies", + "gala_gopher_proc_stime_jiffies", + "gala_gopher_proc_syscall_failed", + "gala_gopher_proc_sched_syscall" + ], + "mem": [ + "gala_gopher_proc_vm_size", + "gala_gopher_proc_pm_size", + "gala_gopher_proc_minor_pagefault_count", + "gala_gopher_proc_referenced_size" + ], + "disk": [ + "gala_gopher_proc_major_pagefault_count", + "gala_gopher_proc_syscr_count", + "gala_gopher_proc_syscw_count", + "gala_gopher_proc_write_bytes", + "gala_gopher_proc_read_bytes", + "gala_gopher_proc_greater_4k_io_write", + "gala_gopher_proc_less_4k_io_write", + "gala_gopher_proc_iowait_us" + ], + "nic": [ + "gala_gopher_proc_ns_sendmsg", + "gala_gopher_proc_ns_recvmsg" + ], + "tcp": [], + "fs": [ + "gala_gopher_proc_fd_count", + "gala_gopher_proc_rchar_bytes", + "gala_gopher_proc_wchar_bytes" + ] + }, + "args": { + "data_dir": "dataset", + "anomaly_topn": 10, + "smooth_window": 12, + "forward_extended_time": 480, + "backward_extended_time": 120, + "maxlag": 2, + "p_threshold": 0.05, + "front_end_metric": "kafka_topic_partition_current_offset", + "corr_type": "pearson", + "corr_prop": 0.3, + "r": 0.8, + "beta": 1, + "num_loop": 5000, + "remove_kpi": "True", + "ac_k": 3 + }, + "metrics": [ + "gala_gopher_sli_rtt_nsec", + "gala_gopher_sli_tps", + "gala_gopher_l7_latency_avg", + "gala_gopher_l7_latency_sum", + "gala_gopher_l7_throughput_req", + "gala_gopher_l7_throughput_resp", + "gala_gopher_l7_err_ratio", + "gala_gopher_l7_err_count", + "gala_gopher_proc_flow_perf_tx_delay", + "gala_gopher_proc_flow_perf_rx_delay", + "gala_gopher_block_count_latency_req", + "gala_gopher_block_latency_device_max", + "gala_gopher_block_latency_device_sum", + "gala_gopher_block_latency_driver_max", + "gala_gopher_block_latency_driver_sum", + "gala_gopher_block_latency_req_max", + "gala_gopher_block_latency_req_sum", + "gala_gopher_disk_aqu", + "gala_gopher_disk_r_await", + "gala_gopher_disk_rareq", + "gala_gopher_disk_rspeed", + "gala_gopher_disk_rspeed_kB", + "gala_gopher_disk_util", + "gala_gopher_disk_w_await", + "gala_gopher_disk_wareq", + "gala_gopher_disk_wspeed", + "gala_gopher_disk_wspeed_kB", + "gala_gopher_cpu_iowait_msec", + "gala_gopher_cpu_irq_msec", + "gala_gopher_cpu_net_rx", + "gala_gopher_cpu_rcu", + "gala_gopher_cpu_sched", + "gala_gopher_cpu_softirq_msec", + "gala_gopher_cpu_system_msec", + "gala_gopher_cpu_timer", + "gala_gopher_cpu_total_used_per", + "gala_gopher_cpu_user_msec", + "gala_gopher_mem_active_kB", + "gala_gopher_mem_cache_kB", + "gala_gopher_mem_free_kB", + "gala_gopher_mem_inactive_kB", + "gala_gopher_mem_util", + "gala_gopher_mem_dentry", + "gala_gopher_proc_bio_latency", + "gala_gopher_proc_iowait_us", + "gala_gopher_proc_bio_err_count", + "gala_gopher_proc_fd_count", + "gala_gopher_proc_fd_free_per", + "gala_gopher_proc_greater_4k_io_read", + "gala_gopher_proc_greater_4k_io_write", + "gala_gopher_proc_less_4k_io_write", + "gala_gopher_proc_less_4k_io_read", + "gala_gopher_proc_minor_pagefault_count", + "gala_gopher_proc_major_pagefault_count", + "gala_gopher_proc_rchar_bytes", + "gala_gopher_proc_read_bytes", + "gala_gopher_proc_pm_size", + "gala_gopher_proc_private_clean_size", + "gala_gopher_proc_private_dirty_size", + "gala_gopher_proc_shared_clean_size", + "gala_gopher_proc_shared_dirty_size", + "gala_gopher_proc_swap_data_size", + "gala_gopher_proc_vm_size", + "gala_gopher_proc_wchar_bytes", + "gala_gopher_proc_write_bytes", + "gala_gopher_proc_offcpu_ns", + "gala_gopher_proc_usr_cpu_ratio", + "gala_gopher_proc_sys_cpu_ratio", + "gala_gopher_tcp_link_rcv_rtt", + "gala_gopher_tcp_link_srtt", + "gala_gopher_tcp_link_retran_packets", + "gala_gopher_tcp_link_avl_snd_wnd", + "gala_gopher_tcp_link_lost_out", + "gala_gopher_tcp_link_rcv_wnd", + "gala_gopher_tcp_link_snd_cwnd", + "gala_gopher_tcp_link_snd_wnd", + "gala_gopher_tcp_link_zero_snd_wnd_count", + "gala_gopher_tcp_link_zero_rcv_wnd_count", + "gala_gopher_tcp_link_reordering", + "gala_gopher_tcp_link_sacked_out", + "gala_gopher_tcp_link_sk_drops", + "gala_gopher_tcp_link_backlog_drops", + "gala_gopher_endpoint_tcp_active_open_failed", + "gala_gopher_endpoint_tcp_passive_open_failed", + "gala_gopher_endpoint_tcp_rst_sent", + "gala_gopher_endpoint_tcp_rst_recv", + "gala_gopher_endpoint_tcp_retran_syn", + "gala_gopher_endpoint_tcp_retran_synacks", + "gala_gopher_dns_max_delay", + "gala_gopher_dns_error_ratio", + "gala_gopher_jvm_mem_bytes_used", + "gala_gopher_jvm_mem_pool_bytes_used", + "gala_gopher_jvm_buffer_pool_used_bytes", + "gala_gopher_jvm_gc_coll_secs_sum", + "gala_gopher_container_cpu_usage_seconds_total", + "gala_gopher_container_cpu_system_seconds_total", + "gala_gopher_container_cpu_user_seconds_total", + "gala_gopher_container_memory_mapped_file", + "gala_gopher_container_memory_cache", + "gala_gopher_container_memory_rss", + "gala_gopher_container_memory_working_set_bytes", + "gala_gopher_container_memory_usage_bytes", + "gala_gopher_container_fs_reads_bytes_total", + "gala_gopher_container_fs_writes_bytes_total", + "gala_gopher_container_fs_read_seconds_total", + "gala_gopher_container_fs_write_seconds_total", + "gala_gopher_container_fs_inodes_free", + "gala_gopher_container_fs_inodes_total", + "gala_gopher_nic_tx_dropped", + "gala_gopher_nic_rx_dropped", + "gala_gopher_nic_tc_enc_mark", + "gala_gopher_nic_tc_sent_drop", + "gala_gopher_jvm_info", + "gala_gopher_jvm_mem_bytes_max", + "gala_gopher_jvm_gc_coll_secs_count", + "gala_gopher_jvm_mem_bytes_commit", + "gala_gopher_jvm_mem_pool_bytes_max", + "gala_gopher_jvm_buffer_pool_capacity_bytes", + "gala_gopher_jvm_class_current_loaded", + "gala_gopher_jvm_threads_current", + "gala_gopher_block_read_bytes", + "gala_gopher_block_write_bytes", + "gala_gopher_tcp_link_notsent_bytes", + "gala_gopher_tcp_link_notack_bytes" + ] +} \ No newline at end of file diff --git a/deploy/en/chart/agents/configs/rca-agent/request_config.json b/deploy/en/chart/agents/configs/rca-agent/request_config.json new file mode 100644 index 000000000..1b5339d7c --- /dev/null +++ b/deploy/en/chart/agents/configs/rca-agent/request_config.json @@ -0,0 +1,4 @@ +{ + "host": "0.0.0.0", + "port": "20030" +} \ No newline at end of file diff --git a/deploy/en/chart/agents/configs/tune/.env.yaml b/deploy/en/chart/agents/configs/tune/.env.yaml new file mode 100644 index 000000000..925c1fa88 --- /dev/null +++ b/deploy/en/chart/agents/configs/tune/.env.yaml @@ -0,0 +1,15 @@ +# Fastapi +UVICORN_IP: 0.0.0.0 +UVICORN_PORT: 8100 + +LLM_KEY: {{ .Values.agents.tune.llm.key }} +LLM_URL: {{ .Values.agents.tune.llm.url }} +LLM_MODEL_NAME: {{ .Values.agents.tune.llm.name }} +LLM_MAX_TOKENS: {{ .Values.agents.tune.llm.max_tokens }} + +# embedding +REMOTE_EMBEDDING_ENDPOINT: {{ .Values.agents.tune.embedding }} + +servers: + - ip: {{ .Values.agents.tune.machine.ip }} + password: {{ .Values.agents.tune.machine.password }} \ No newline at end of file diff --git a/deploy/en/chart/agents/configs/tune/app_config.yaml b/deploy/en/chart/agents/configs/tune/app_config.yaml new file mode 100644 index 000000000..61f7ed7ec --- /dev/null +++ b/deploy/en/chart/agents/configs/tune/app_config.yaml @@ -0,0 +1,3 @@ +MySQL: + user: {{ .Values.agents.tune.mysql.user }} + password: {{ .Values.agents.tune.mysql.password }} \ No newline at end of file diff --git a/deploy/en/chart/agents/configs/tune/knob_rag_config.json b/deploy/en/chart/agents/configs/tune/knob_rag_config.json new file mode 100644 index 000000000..37d580487 --- /dev/null +++ b/deploy/en/chart/agents/configs/tune/knob_rag_config.json @@ -0,0 +1,4 @@ +{ + "threshold": 0.5, + "topk": 10 +} \ No newline at end of file diff --git a/deploy/en/chart/agents/configs/tune/optimize_config.yaml b/deploy/en/chart/agents/configs/tune/optimize_config.yaml new file mode 100644 index 000000000..65ed37c48 --- /dev/null +++ b/deploy/en/chart/agents/configs/tune/optimize_config.yaml @@ -0,0 +1,3 @@ +knob_tuning : "static" +evaluations : "" +goal : "" \ No newline at end of file diff --git a/deploy/en/chart/agents/templates/NOTES.txt b/deploy/en/chart/agents/templates/NOTES.txt new file mode 100644 index 000000000..2b076bd6b --- /dev/null +++ b/deploy/en/chart/agents/templates/NOTES.txt @@ -0,0 +1 @@ +Thank you for choosing Euler Copilot! diff --git a/deploy/en/chart/agents/templates/ai-infra-agent/ai-infra-deployment.yaml b/deploy/en/chart/agents/templates/ai-infra-agent/ai-infra-deployment.yaml new file mode 100644 index 000000000..9b680c178 --- /dev/null +++ b/deploy/en/chart/agents/templates/ai-infra-agent/ai-infra-deployment.yaml @@ -0,0 +1,55 @@ +{{- if .Values.agents.ai_infra.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ai-infra-deploy-{{ .Release.Name }} + namespace: {{ .Release.Namespace }} + labels: + app: ai-infra-{{ .Release.Name }} +spec: + replicas: 1 + selector: + matchLabels: + app: ai-infra-{{ .Release.Name }} + template: + metadata: + annotations: + checksum/secret: {{ include (print $.Template.BasePath "/ai-infra-agent/ai-infra-secret.yaml") . | sha256sum }} + labels: + app: ai-infra-{{ .Release.Name }} + spec: + automountServiceAccountToken: false + containers: + - name: ai-infra + image: "{{if ne ( .Values.agents.ai_infra.image.registry | toString ) ""}}{{ .Values.agents.ai_infra.image.registry }}{{ else }}{{ .Values.globals.imageRegistry }}{{ end }}/{{ .Values.agents.ai_infra.image.name }}:{{ .Values.agents.ai_infra.image.tag | toString }}" + imagePullPolicy: {{ if ne ( .Values.agents.ai_infra.image.imagePullPolicy | toString ) "" }}{{ .Values.agents.ai_infra.image.imagePullPolicy }}{{ else }}{{ .Values.globals.imagePullPolicy }}{{ end }} + ports: + - containerPort: 8101 + protocol: TCP + env: + - name: TZ + value: Asia/Shanghai + - name: PYTHONPATH + value: /app + volumeMounts: + - mountPath: /app/config + name: ai-infra-secret-volume + securityContext: + readOnlyRootFilesystem: {{ .Values.agents.ai_infra.readOnly }} + capabilities: + drop: + - ALL + runAsUser: 1001 + runAsGroup: 1001 + runAsNonRoot: true + allowPrivilegeEscalation: false + resources: + {{- toYaml .Values.agents.ai_infra.resources | nindent 12 }} + volumes: + - name: ai-infra-secret-volume + secret: + secretName: ai-infra-secret-{{ .Release.Name }} + items: + - key: .env.yaml + path: .env.yaml +{{- end }} diff --git a/deploy/en/chart/agents/templates/ai-infra-agent/ai-infra-secret.yaml b/deploy/en/chart/agents/templates/ai-infra-agent/ai-infra-secret.yaml new file mode 100644 index 000000000..ba7c3ea75 --- /dev/null +++ b/deploy/en/chart/agents/templates/ai-infra-agent/ai-infra-secret.yaml @@ -0,0 +1,11 @@ +{{- if .Values.agents.ai_infra.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: ai-infra-secret-{{ .Release.Name }} + namespace: {{ .Release.Namespace }} +type: Opaque +stringData: + .env.yaml: | +{{ tpl (.Files.Get "configs/ai-infra-agent/.env.yaml") . | indent 4 }} +{{- end }} diff --git a/deploy/en/chart/agents/templates/ai-infra-agent/ai-infra-service.yaml b/deploy/en/chart/agents/templates/ai-infra-agent/ai-infra-service.yaml new file mode 100644 index 000000000..afa61529f --- /dev/null +++ b/deploy/en/chart/agents/templates/ai-infra-agent/ai-infra-service.yaml @@ -0,0 +1,17 @@ +{{- if .Values.agents.ai_infra.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: ai-infra-service-{{ .Release.Name }} + namespace: {{ .Release.Namespace }} +spec: + type: {{ .Values.agents.ai_infra.service.type }} + selector: + app: ai-infra-{{ .Release.Name }} + ports: + - port: 8101 + targetPort: 8101 + {{- if (and (eq .Values.agents.ai_infra.service.type "") .Values.agents.ai_infra.service.nodePort) }} + nodePort: {{ .Values.agents.ai_infra.service.nodePort }} + {{- end }} +{{- end }} diff --git a/deploy/en/chart/agents/templates/rca-agent/rca-agent-config.yaml b/deploy/en/chart/agents/templates/rca-agent/rca-agent-config.yaml new file mode 100644 index 000000000..a2efca471 --- /dev/null +++ b/deploy/en/chart/agents/templates/rca-agent/rca-agent-config.yaml @@ -0,0 +1,12 @@ +{{- if .Values.agents.rca.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: rca-agent-config-{{ .Release.Name }} + namespace: {{ .Release.Namespace }} +data: + request_config.json: |- +{{ tpl (.Files.Get "configs/rca-agent/request_config.json") . | indent 4 }} + config.json: |- +{{ tpl (.Files.Get "configs/rca-agent/config.json") . | indent 4 }} +{{- end }} \ No newline at end of file diff --git a/deploy/en/chart/agents/templates/rca-agent/rca-agent-deployment.yaml b/deploy/en/chart/agents/templates/rca-agent/rca-agent-deployment.yaml new file mode 100644 index 000000000..df41a69a2 --- /dev/null +++ b/deploy/en/chart/agents/templates/rca-agent/rca-agent-deployment.yaml @@ -0,0 +1,51 @@ +{{- if .Values.agents.rca.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rca-agent-{{ .Release.Name }} + namespace: {{ .Release.Namespace }} + labels: + app: rca-agent-{{ .Release.Name }} +spec: + replicas: 1 + selector: + matchLabels: + app: rca-agent-{{ .Release.Name }} + template: + metadata: + labels: + app: rca-agent-{{ .Release.Name }} + spec: + automountServiceAccountToken: false + containers: + - name: rca-agent + image: "{{if ne ( .Values.agents.rca.image.registry | toString ) ""}}{{ .Values.agents.rca.image.registry }}{{ else }}{{ .Values.globals.imageRegistry }}{{ end }}/{{ .Values.agents.rca.image.name }}:{{ .Values.agents.rca.image.tag | toString }}" + imagePullPolicy: {{ if ne ( .Values.agents.rca.image.imagePullPolicy | toString ) "" }}{{ .Values.agents.rca.image.imagePullPolicy }}{{ else }}{{ .Values.globals.imagePullPolicy }}{{ end }} + ports: + - containerPort: 20030 + protocol: TCP + env: + - name: TZ + value: Asia/Shanghai + volumeMounts: + - mountPath: /var/log/gala-gopher + name: gopher-profile + - mountPath: /home/euler-copilot-rca/config/request_config.json + name: rca-config-volume + subPath: request_config.json + - mountPath: /home/euler-copilot-rca/config/config.json + name: rca-config-volume + subPath: config.json + securityContext: + readOnlyRootFilesystem: {{ .Values.agents.rca.readOnly }} + resources: + {{- toYaml .Values.agents.rca.resources | nindent 12 }} + volumes: + - name: rca-config-volume + configMap: + name: rca-agent-config-{{ .Release.Name }} + - name: gopher-profile + hostPath: + path: /var/log/gala-gopher + type: DirectoryOrCreate +{{- end }} \ No newline at end of file diff --git a/deploy/en/chart/agents/templates/rca-agent/rca-agent-service.yaml b/deploy/en/chart/agents/templates/rca-agent/rca-agent-service.yaml new file mode 100644 index 000000000..8cf69958e --- /dev/null +++ b/deploy/en/chart/agents/templates/rca-agent/rca-agent-service.yaml @@ -0,0 +1,17 @@ +{{- if .Values.agents.rca.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: rca-agent-service-{{ .Release.Name }} + namespace: {{ .Release.Namespace }} +spec: + type: {{ .Values.agents.rca.service.type }} + selector: + app: rca-agent-{{ .Release.Name }} + ports: + - port: 20030 + targetPort: 20030 + {{- if (and (eq .Values.agents.rca.service.type "") .Values.agents.rca.service.nodePort) }} + nodePort: {{ .Values.agents.rca.service.nodePort }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/deploy/en/chart/agents/templates/tune/tune-deployment.yaml b/deploy/en/chart/agents/templates/tune/tune-deployment.yaml new file mode 100644 index 000000000..532464401 --- /dev/null +++ b/deploy/en/chart/agents/templates/tune/tune-deployment.yaml @@ -0,0 +1,52 @@ +{{- if .Values.agents.tune.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tune-deploy-{{ .Release.Name }} + namespace: {{ .Release.Namespace }} + labels: + app: tune-{{ .Release.Name }} +spec: + replicas: 1 + selector: + matchLabels: + app: tune-{{ .Release.Name }} + template: + metadata: + labels: + app: tune-{{ .Release.Name }} + spec: + automountServiceAccountToken: false + containers: + - name: tune + image: "{{if ne ( .Values.agents.tune.image.registry | toString ) ""}}{{ .Values.agents.tune.image.registry }}{{ else }}{{ .Values.globals.imageRegistry }}{{ end }}/{{ .Values.agents.tune.image.name }}:{{ .Values.agents.tune.image.tag | toString }}" + imagePullPolicy: {{ if ne ( .Values.agents.tune.image.imagePullPolicy | toString ) "" }}{{ .Values.agents.tune.image.imagePullPolicy }}{{ else }}{{ .Values.globals.imagePullPolicy }}{{ end }} + ports: + - containerPort: 8100 + protocol: TCP + env: + - name: TZ + value: Asia/Shanghai + - name: PYTHONPATH + value: /app + volumeMounts: + - mountPath: /app/config + name: tune-secret-volume + securityContext: + readOnlyRootFilesystem: {{ .Values.agents.tune.readOnly }} + resources: + {{- toYaml .Values.agents.tune.resources | nindent 12 }} + volumes: + - name: tune-secret-volume + secret: + secretName: tune-secret-{{ .Release.Name }} + items: + - key: .env.yaml + path: .env.yaml + - key: app_config.yaml + path: app_config.yaml + - key: knob_rag_config.json + path: knob_rag_config.json + - key: optimize_config.yaml + path: optimize_config.yaml +{{- end }} diff --git a/deploy/en/chart/agents/templates/tune/tune-secret.yaml b/deploy/en/chart/agents/templates/tune/tune-secret.yaml new file mode 100644 index 000000000..aced99207 --- /dev/null +++ b/deploy/en/chart/agents/templates/tune/tune-secret.yaml @@ -0,0 +1,17 @@ +{{- if .Values.agents.tune.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: tune-secret-{{ .Release.Name }} + namespace: {{ .Release.Namespace }} +type: Opaque +stringData: + .env.yaml: |- +{{ tpl (.Files.Get "configs/tune/.env.yaml") . | indent 4 }} + app_config.yaml: |- +{{ tpl (.Files.Get "configs/tune/app_config.yaml") . | indent 4 }} + knob_rag_config.json: |- +{{ tpl (.Files.Get "configs/tune/knob_rag_config.json") . | indent 4 }} + optimize_config.yaml: |- +{{ tpl (.Files.Get "configs/tune/optimize_config.yaml") . | indent 4 }} +{{- end }} \ No newline at end of file diff --git a/deploy/en/chart/agents/templates/tune/tune-service.yaml b/deploy/en/chart/agents/templates/tune/tune-service.yaml new file mode 100644 index 000000000..f2d8f7aee --- /dev/null +++ b/deploy/en/chart/agents/templates/tune/tune-service.yaml @@ -0,0 +1,17 @@ +{{- if .Values.agents.tune.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: tune-service-{{ .Release.Name }} + namespace: {{ .Release.Namespace }} +spec: + type: {{ .Values.agents.tune.service.type }} + selector: + app: tune-{{ .Release.Name }} + ports: + - port: 8100 + targetPort: 8100 + {{- if (and (eq .Values.agents.tune.service.type "") .Values.agents.tune.service.nodePort) }} + nodePort: {{ .Values.agents.tune.service.nodePort }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/deploy/en/chart/agents/values.yaml b/deploy/en/chart/agents/values.yaml new file mode 100644 index 000000000..b7da9d16d --- /dev/null +++ b/deploy/en/chart/agents/values.yaml @@ -0,0 +1,100 @@ +# Global Settings +globals: + # [Required] Image Registry + imageRegistry: "hub.oepkgs.net/neocopilot" + # [Required] Image Pull Policy + imagePullPolicy: IfNotPresent + +# OpenEuler Product Agents +agents: + ai_infra: + # [Required] Enable AI Container Image Agent + enabled: true + # Image Settings + image: + # Image registry. Leave empty to use global settings. + registry: "" + # [Required] Image name + name: compatibility-ai-infra + # [Required] Image Tag + tag: "0.9.1" + # Pull policy. Leave empty to use global settings + imagePullPolicy: "" + # [Required] Container root directory read-only + readOnly: true + # Resource limits settings + resources: {} + # Service Settings + service: + # [Required] Service type, ClusterIP or NodePort + type: ClusterIP + nodePort: + rca: + # [Required] Enable Intelligent Diagnostics Agent. Must be used with intelligent diagnostics server. + enabled: true + # Image Settings + image: + # Image registry. Leave empty to use global settings. + registry: "hub.oepkgs.net/a-ops" + # [Required] Image name + name: euler-copilot-rca + # [Required] Image tag + tag: "0.9.1" + # Pull policy. Leave empty to use global settings. + imagePullPolicy: "" + # [Required] Container root directory read-only + readOnly: true + # Resource limits settings + resources: {} + # Service Settings + service: + # [Required] Service type, ClusterIP or NodePort + type: ClusterIP + nodePort: + tune: + # [Required] Enable Intelligent Tuning Agent. + enabled: true + # Image Settings + image: + # Image registry. Leave empty to use global settings. + registry: "" + # [Required] Image name + name: euler-copilot-tune + # [Required] Image tag + tag: "0.9.1" + # Pull policy. Leave empty to use global settings. + imagePullPolicy: "" + # [Required] Container root directory read-only + readOnly: true + # Resource limits settings + resources: {} + # Service Settings + service: + # [Required] Service type, ClusterIP or NodePort + type: ClusterIP + nodePort: + # Large Language Model Settings + llm: + # [Required] Model URL (must include v1 suffix) + url: + # [Required] Model name + name: "" + # [Required] Model API Key + key: "" + # [Required] Model maximum tokens + max_tokens: 8096 + # [Required] Embedding URL + embedding: "" + # Target machine information for optimization + machine: + # [Required] IP address + ip: "" + # [Required] Root user password + # Note: Must enable Root user SSH login with password + password: "" + # Target application settings for optimization + mysql: + # [Required] Database username + user: "root" + # [Required] Database password + password: "" diff --git a/deploy/en/chart/authhub/.helmignore b/deploy/en/chart/authhub/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/deploy/en/chart/authhub/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/deploy/en/chart/authhub/Chart.yaml b/deploy/en/chart/authhub/Chart.yaml new file mode 100644 index 000000000..64e763d73 --- /dev/null +++ b/deploy/en/chart/authhub/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: authhub-chart +description: AuthHub Helm Chart +type: application +version: 0.10.0 +appVersion: "0.10.0" diff --git a/deploy/en/chart/authhub/configs/backend/aops-config.yml b/deploy/en/chart/authhub/configs/backend/aops-config.yml new file mode 100644 index 000000000..c0e1ca44b --- /dev/null +++ b/deploy/en/chart/authhub/configs/backend/aops-config.yml @@ -0,0 +1,30 @@ +infrastructure: + mysql: + host: mysql-db.{{ .Release.Namespace }}.svc.cluster.local + port: 3306 + username: authhub + pool_size: 100 + pool_recycle: 7200 + database: oauth2 + password: ${mysql-password} + redis: + host: redis-db.{{ .Release.Namespace }}.svc.cluster.local + port: 6379 + password: ${redis-password} + +include: "/etc/aops/conf.d" +domain: '{{ regexFind "^[^:]+" (.Values.domain.authhub| default "http://127.0.0.1:30080" | replace "http://" "" | replace "https://" "") }}' +services: + log: + log_level: "INFO" + log_dir: "/var/log/aops" + max_bytes: 31457280 + backup_count: 40 + + email: + server: smtp.163.com + port: 25 + sender: xxx@163.com + authorization_code: xxx + smtp_ssl: false + enabled: false diff --git a/deploy/en/chart/authhub/configs/backend/authhub.yml b/deploy/en/chart/authhub/configs/backend/authhub.yml new file mode 100644 index 000000000..78677bd2e --- /dev/null +++ b/deploy/en/chart/authhub/configs/backend/authhub.yml @@ -0,0 +1,4 @@ +uwsgi: + port: 11120 + processes: 1 + daemonize: /var/log/oauth2/uwsgi/oauthhub.log \ No newline at end of file diff --git a/deploy/en/chart/authhub/configs/mysql/init.sql b/deploy/en/chart/authhub/configs/mysql/init.sql new file mode 100644 index 000000000..9c1b9fc2a --- /dev/null +++ b/deploy/en/chart/authhub/configs/mysql/init.sql @@ -0,0 +1,112 @@ +CREATE DATABASE IF NOT EXISTS oauth2 DEFAULT CHARACTER SET utf8mb4 DEFAULT COLLATE utf8mb4_bin; +GRANT ALL ON `oauth2`.* TO 'authhub'@'%'; +use oauth2; + +SET FOREIGN_KEY_CHECKS = 0; + +CREATE TABLE IF NOT EXISTS `manage_user` ( + `id` int NOT NULL AUTO_INCREMENT, + `username` varchar(36) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + `password` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `username` (`username`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + +CREATE TABLE IF NOT EXISTS `user` ( + `id` int NOT NULL AUTO_INCREMENT, + `username` varchar(36) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + `password` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + `email` varchar(40) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, + `phone` varchar(11) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `username` (`username`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + +CREATE TABLE IF NOT EXISTS `oauth2_client` ( + `id` int NOT NULL AUTO_INCREMENT, + `app_name` varchar(48) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + `username` varchar(36) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, + `client_id` varchar(48) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, + `client_secret` varchar(120) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, + `client_id_issued_at` int NOT NULL, + `client_secret_expires_at` int NOT NULL, + `client_metadata` text, + PRIMARY KEY (`id`), + UNIQUE KEY `app_name` (`app_name`), + UNIQUE KEY `client_id` (`client_id`), + KEY `username` (`username`), + KEY `ix_oauth2_client_client_id` (`client_id`), + CONSTRAINT `oauth2_client_ibfk_1` FOREIGN KEY (`username`) REFERENCES `manage_user` (`username`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + +CREATE TABLE IF NOT EXISTS `login_records` ( + `id` int NOT NULL AUTO_INCREMENT, + `username` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, + `login_time` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, + `client_id` varchar(48) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, + `logout_url` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, + PRIMARY KEY (`id`), + CONSTRAINT `login_records_ibfk_1` FOREIGN KEY (`client_id`) REFERENCES `oauth2_client` (`client_id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + +CREATE TABLE IF NOT EXISTS `oauth2_client_scopes` ( + `id` int NOT NULL AUTO_INCREMENT, + `username` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, + `client_id` int DEFAULT NULL, + `scopes` text CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + `grant_at` int NOT NULL, + `expires_in` int NOT NULL, + PRIMARY KEY (`id`), + CONSTRAINT `oauth2_client_scopes_ibfk_1` FOREIGN KEY (`client_id`) REFERENCES `oauth2_client` (`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + +CREATE TABLE IF NOT EXISTS `oauth2_code` ( + `id` int NOT NULL AUTO_INCREMENT, + `username` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, + `code` varchar(120) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + `client_id` varchar(48) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, + `redirect_uri` text CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, + `response_type` text CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, + `scope` text CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, + `nonce` text CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, + `auth_time` int NOT NULL, + `code_challenge` text CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, + `code_challenge_method` varchar(48) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + +CREATE TABLE IF NOT EXISTS `oauth2_token` ( + `id` int NOT NULL AUTO_INCREMENT, + `user_id` int DEFAULT NULL, + `username` varchar(36) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + `client_id` varchar(48) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + `token_metadata` text CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, + `refresh_token_expires_in` int NOT NULL, + `token_type` varchar(40) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, + `access_token` varchar(4096) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + `refresh_token` varchar(4096) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, + `scope` text CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, + `issued_at` int NOT NULL, + `access_token_revoked_at` int NOT NULL, + `refresh_token_revoked_at` int NOT NULL, + `expires_in` int NOT NULL, + PRIMARY KEY (`id`), + KEY `user_id` (`user_id`), + CONSTRAINT `oauth2_token_ibfk_1` FOREIGN KEY (`user_id`) REFERENCES `user` (`id`) ON DELETE CASCADE, + CONSTRAINT `oauth2_token_ibfk_2` FOREIGN KEY (`client_id`) REFERENCES `oauth2_client` (`client_id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + +SET FOREIGN_KEY_CHECKS = 1; + +SET @username := "admin"; +SET @password := "pbkdf2:sha256:260000$LEwtriXN8UQ1UIA7$4de6cc1d67263c6579907eab7c1cba7c7e857b32e957f9ff5429592529d7d1b0"; +SET @manage_username := "administrator"; + +INSERT INTO user (username, password) +SELECT @username, @password +FROM DUAL +WHERE NOT EXISTS(SELECT 1 FROM user WHERE username = @username); +INSERT INTO manage_user (username, password) +SELECT @manage_username, @password +FROM DUAL +WHERE NOT EXISTS(SELECT 1 FROM manage_user WHERE username = @username); \ No newline at end of file diff --git a/deploy/en/chart/authhub/templates/NOTES.txt b/deploy/en/chart/authhub/templates/NOTES.txt new file mode 100644 index 000000000..896a6441a --- /dev/null +++ b/deploy/en/chart/authhub/templates/NOTES.txt @@ -0,0 +1,7 @@ +Thank you for using openEuler Intelligence! +Current version is 0.10.0. +The current Chart functionality is: AuthHub Unified Login System deployment. + +Notes: +The default username for AuthHub is: administrator +The default password for AuthHub is: changeme diff --git a/deploy/en/chart/authhub/templates/backend/authhub-backend-config.yaml b/deploy/en/chart/authhub/templates/backend/authhub-backend-config.yaml new file mode 100644 index 000000000..0509245f7 --- /dev/null +++ b/deploy/en/chart/authhub/templates/backend/authhub-backend-config.yaml @@ -0,0 +1,32 @@ +{{- if .Values.authhub.backend.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: authhub-backend-config + namespace: {{ .Release.Namespace }} +data: + aops-config.yml: |- +{{ tpl (.Files.Get "configs/backend/aops-config.yml") . | indent 4 }} + authhub.yml: |- +{{ tpl (.Files.Get "configs/backend/authhub.yml") . | indent 4 }} + copy-config.yml: |- + copy: + - from: /config/aops-config.yml + to: /config-rw/aops-config.yml + mode: + uid: 0 + gid: 0 + mode: "0o650" + secrets: + - /db-secrets + - /authhub-secrets + - from: /config/conf.d/authhub.yml + to: /config-rw/conf.d/authhub.yml + mode: + uid: 0 + gid: 0 + mode: "0o650" + secrets: + - /db-secrets + - /authhub-secrets +{{- end -}} diff --git a/deploy/en/chart/authhub/templates/backend/authhub-backend.yaml b/deploy/en/chart/authhub/templates/backend/authhub-backend.yaml new file mode 100644 index 000000000..60246589b --- /dev/null +++ b/deploy/en/chart/authhub/templates/backend/authhub-backend.yaml @@ -0,0 +1,101 @@ +{{- if .Values.authhub.backend.enabled -}} +--- +apiVersion: v1 +kind: Service +metadata: + name: authhub-backend-service + namespace: {{ .Release.Namespace }} +spec: + type: {{ default "ClusterIP" .Values.authhub.backend.service.type }} + selector: + app: authhub-backend + ports: + - port: 11120 + targetPort: 11120 + nodePort: {{ default nil .Values.authhub.backend.service.nodePort }} + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: authhub-backend-deploy + namespace: {{ .Release.Namespace }} + labels: + app: authhub-backend +spec: + replicas: {{ default 1 .Values.globals.replicaCount }} + selector: + matchLabels: + app: authhub-backend + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/backend/authhub-backend-config.yaml") . | sha256sum }} + labels: + app: authhub-backend + spec: + automountServiceAccountToken: false + containers: + - name: authhub-backend + image: {{ .Values.authhub.backend.image | default (printf "%s/neocopilot/authhub:0.9.3-%s" (.Values.globals.imageRegistry | default "hub.oepkgs.net") (.Values.globals.arch | default "x86")) }} + imagePullPolicy: {{ default "IfNotPresent" .Values.globals.imagePullPolicy }} + ports: + - containerPort: 11120 + protocol: TCP + volumeMounts: + - name: authhub-shared + mountPath: /etc/aops + livenessProbe: + httpGet: + path: /oauth2/applications + port: 11120 + scheme: HTTP + failureThreshold: 5 + initialDelaySeconds: 60 + periodSeconds: 90 + resources: + requests: + cpu: 0.1 + memory: 128Mi + limits: + {{ toYaml .Values.authhub.backend.resourceLimits | nindent 14 }} + initContainers: + - name: authback-copy + image: {{ .Values.authhub.secret_inject.image | default (printf "%s/neocopilot/secret_inject:dev-%s" (.Values.globals.imageRegistry | default "hub.oepkgs.net") (.Values.globals.arch | default "x86")) }} + imagePullPolicy: {{ default "IfNotPresent" .Values.globals.imagePullPolicy }} + command: + - python3 + - ./main.py + - --config + - /config/copy-config.yml + - --copy + volumeMounts: + - mountPath: /db-secrets + name: euler-copilot-database-vl + - mountPath: /authhub-secrets + name: authhub-secret-vl + - mountPath: /config-rw + name: authhub-shared + - mountPath: /config + name: authhub-config + volumes: + - name: authhub-shared + emptyDir: + medium: Memory + - name: authhub-config + configMap: + name: authhub-backend-config + items: + - key: aops-config.yml + path: aops-config.yml + - key: authhub.yml + path: conf.d/authhub.yml + - key: copy-config.yml + path: copy-config.yml + - name: authhub-secret-vl + secret: + secretName: authhub-secret + - name: euler-copilot-database-vl + secret: + secretName: euler-copilot-database +{{- end -}} diff --git a/deploy/en/chart/authhub/templates/mysql/mysql-config.yaml b/deploy/en/chart/authhub/templates/mysql/mysql-config.yaml new file mode 100644 index 000000000..9a92389af --- /dev/null +++ b/deploy/en/chart/authhub/templates/mysql/mysql-config.yaml @@ -0,0 +1,10 @@ +{{- if .Values.authhub.mysql.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: mysql-config + namespace: {{ .Release.Namespace }} +data: + init.sql: |- +{{ tpl (.Files.Get "configs/mysql/init.sql") . | indent 4 }} +{{- end -}} \ No newline at end of file diff --git a/deploy/en/chart/authhub/templates/mysql/mysql-storage.yaml b/deploy/en/chart/authhub/templates/mysql/mysql-storage.yaml new file mode 100644 index 000000000..0677d68a7 --- /dev/null +++ b/deploy/en/chart/authhub/templates/mysql/mysql-storage.yaml @@ -0,0 +1,16 @@ +{{- if .Values.authhub.mysql.enabled -}} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: mysql-pvc + namespace: {{ .Release.Namespace }} + annotations: + helm.sh/resource-policy: keep +spec: + storageClassName: {{ default "local-path" .Values.globals.storageClassName }} + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ default "10Gi" .Values.authhub.mysql.persistentVolumeSize }} +{{- end -}} \ No newline at end of file diff --git a/deploy/en/chart/authhub/templates/mysql/mysql.yaml b/deploy/en/chart/authhub/templates/mysql/mysql.yaml new file mode 100644 index 000000000..098ee79c2 --- /dev/null +++ b/deploy/en/chart/authhub/templates/mysql/mysql.yaml @@ -0,0 +1,90 @@ +{{- if .Values.authhub.mysql.enabled -}} +--- +apiVersion: v1 +kind: Service +metadata: + name: mysql-db + namespace: {{ .Release.Namespace }} +spec: + type: {{ default "ClusterIP" .Values.authhub.mysql.service.type }} + selector: + app: mysql + ports: + - port: 3306 + targetPort: 3306 + nodePort: {{ default nil .Values.authhub.mysql.service.nodePort }} + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mysql-deploy + namespace: {{ .Release.Namespace }} + labels: + app: mysql +spec: + replicas: {{ default 1 .Values.globals.replicaCount }} + selector: + matchLabels: + app: mysql + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/mysql/mysql-config.yaml") . | sha256sum }} + labels: + app: mysql + spec: + automountServiceAccountToken: false + containers: + - name: mysql + image: {{ .Values.authhub.mysql.image | default (printf "%s/neocopilot/mysql:8-%s" (.Values.globals.imageRegistry | default "hub.oepkgs.net") (.Values.globals.arch | default "x86")) }} + imagePullPolicy: {{ default "IfNotPresent" .Values.globals.imagePullPolicy }} + args: + - "--character-set-server=utf8mb4" + - "--collation-server=utf8mb4_unicode_ci" + ports: + - containerPort: 3306 + protocol: TCP + livenessProbe: + exec: + command: + - sh + - -c + - mysqladmin -h 127.0.0.1 -u $MYSQL_USER --password=$MYSQL_PASSWORD ping + failureThreshold: 5 + initialDelaySeconds: 60 + periodSeconds: 90 + env: + - name: TZ + value: "Asia/Shanghai" + - name: MYSQL_USER + value: "authhub" + - name: MYSQL_RANDOM_ROOT_PASSWORD + value: "yes" + - name: MYSQL_PASSWORD + valueFrom: + secretKeyRef: + name: authhub-secret + key: mysql-password + volumeMounts: + - mountPath: /var/lib/mysql + name: mysql-data + - mountPath: /docker-entrypoint-initdb.d/init.sql + name: mysql-init + subPath: init.sql + resources: + requests: + cpu: 0.1 + memory: 384Mi + limits: + {{ toYaml .Values.authhub.mysql.resourceLimits | nindent 14 }} + restartPolicy: Always + volumes: + - name: mysql-data + persistentVolumeClaim: + claimName: mysql-pvc + - name: mysql-init + configMap: + name: mysql-config +{{- end -}} + diff --git a/deploy/en/chart/authhub/templates/secrets.yaml b/deploy/en/chart/authhub/templates/secrets.yaml new file mode 100644 index 000000000..a274f0973 --- /dev/null +++ b/deploy/en/chart/authhub/templates/secrets.yaml @@ -0,0 +1,24 @@ +{{- $authhubSecret := (lookup "v1" "Secret" .Release.Namespace "authhub-secret") -}} +{{- if $authhubSecret -}} +apiVersion: v1 +kind: Secret +metadata: + name: authhub-secret + namespace: {{ .Release.Namespace }} + annotations: + helm.sh/resource-policy: keep +type: Opaque +stringData: + mysql-password: {{ index $authhubSecret.data "mysql-password" | b64dec }} +{{- else -}} +apiVersion: v1 +kind: Secret +metadata: + name: authhub-secret + namespace: {{ .Release.Namespace }} + annotations: + helm.sh/resource-policy: keep +type: Opaque +stringData: + mysql-password: {{ randAlphaNum 20 }} +{{- end -}} \ No newline at end of file diff --git a/deploy/en/chart/authhub/templates/web/authhub-web-config.yaml b/deploy/en/chart/authhub/templates/web/authhub-web-config.yaml new file mode 100644 index 000000000..3d342fb6e --- /dev/null +++ b/deploy/en/chart/authhub/templates/web/authhub-web-config.yaml @@ -0,0 +1,44 @@ +{{- if .Values.authhub.web.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: authhub-web-config + namespace: {{ .Release.Namespace }} +data: + authhub.nginx.conf: |- + server { + listen 8000; + server_name localhost; + + # gzip config + gzip on; + gzip_min_length 1k; + gzip_comp_level 6; + gzip_types text/plain text/css text/javascript application/json application/javascript application/x-javascript application/xml; + gzip_vary on; + gzip_disable "MSIE [1-6]\."; + + location / { + proxy_set_header X-Real-IP $remote_addr; + root /opt/authhub/web/dist; + index index.html; + try_files $uri $uri/ /index.html; + } + + location /authhub { + add_header Access-Control-Allow-Origin *; + add_header Access-Control-Allow-Methods 'GET, POST, DELETE, PUT, OPTIONS'; + alias /opt/authhub/web/dist; + index index.html; + try_files $uri $uri/ /index.html last; + } + + location /oauth2 { + proxy_pass http://authhub-backend-service.{{ .Release.Namespace }}.svc.cluster.local:11120; + proxy_set_header Host $host; + proxy_set_header X-Real-URL $request_uri; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Request-Header $http_request_header; + } + } +{{- end -}} \ No newline at end of file diff --git a/deploy/en/chart/authhub/templates/web/authhub-web.yaml b/deploy/en/chart/authhub/templates/web/authhub-web.yaml new file mode 100644 index 000000000..91bec2d9b --- /dev/null +++ b/deploy/en/chart/authhub/templates/web/authhub-web.yaml @@ -0,0 +1,67 @@ +{{- if .Values.authhub.web.enabled -}} +--- +apiVersion: v1 +kind: Service +metadata: + name: authhub-web-service + namespace: {{ .Release.Namespace }} +spec: + type: {{ default "NodePort" .Values.authhub.web.service.type }} + selector: + app: authhub-web + ports: + - port: 8000 + targetPort: 8000 + nodePort: {{ default 30081 .Values.authhub.web.service.nodePort }} + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: authhub-web-deploy + namespace: {{ .Release.Namespace }} + labels: + app: authhub-web +spec: + replicas: {{ default 1 .Values.globals.replicaCount }} + selector: + matchLabels: + app: authhub-web + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/web/authhub-web-config.yaml") . | sha256sum }} + labels: + app: authhub-web + spec: + automountServiceAccountToken: false + containers: + - name: authhub-web + image: {{ .Values.authhub.web.image | default (printf "%s/neocopilot/authhub-web:0.9.3-%s" (.Values.globals.imageRegistry | default "hub.oepkgs.net") (.Values.globals.arch | default "x86")) }} + imagePullPolicy: {{ default "IfNotPresent" .Values.globals.imagePullPolicy }} + ports: + - containerPort: 8000 + protocol: TCP + livenessProbe: + httpGet: + path: / + port: 8000 + scheme: HTTP + failureThreshold: 5 + initialDelaySeconds: 60 + periodSeconds: 90 + volumeMounts: + - name: web-config + mountPath: /etc/nginx/conf.d/authhub.nginx.conf + subPath: authhub.nginx.conf + resources: + requests: + cpu: 0.05 + memory: 64Mi + limits: + {{ toYaml .Values.authhub.web.resourceLimits | nindent 14 }} + volumes: + - name: web-config + configMap: + name: authhub-web-config +{{- end -}} diff --git a/deploy/en/chart/authhub/values.yaml b/deploy/en/chart/authhub/values.yaml new file mode 100644 index 000000000..fd2e249a3 --- /dev/null +++ b/deploy/en/chart/authhub/values.yaml @@ -0,0 +1,72 @@ +# Global Settings +globals: + # Node Architecture: default is x86 + # [Required] Node settings: ["x86", "arm"] + arch: + # Image pull policy; default is IfNotPresent + imagePullPolicy: + # Replica count, default is 1 + replicaCount: + # Storage class name; default is local-path + storageClassName: + +storage: + # MySQL persistent storage size, default is 10Gi + mysql: + +domain: + # [Required] AuthHub web URL, default is http://127.0.0.1:30081 + authhub: + +# Deploy AuthHub local authentication service +authhub: + # Configuration file tool + secret_inject: + # Image settings; default is hub.oepkgs.net/neocopilot/secret_inject:dev-x86 + # Image tags: ["dev-x86", "dev-arm"] + image: + + web: + # [Required] Whether to deploy AuthHub frontend service + enabled: true + # Image settings; default is hub.oepkgs.net/neocopilot/authhub-web:0.9.3-x86 + # Image tags: ["0.9.3-x86", "0.9.3-arm"] + image: + # Resource limits settings + resourceLimits: {} + # Service settings + service: + # Service type, e.g., NodePort + type: NodePort + # When type is NodePort, specify the host port number + nodePort: 30081 + + backend: + # [Required] Whether to deploy AuthHub backend service + enabled: true + # Image settings; default is hub.oepkgs.net/neocopilot/authhub:0.9.3-x86 + # Image tags: ["0.9.3-x86", "0.9.3-arm"] + image: + # Resource limits settings + resourceLimits: {} + # Service settings + service: + # Service type, e.g., NodePort + type: + # When type is NodePort, specify the host port number + nodePort: + + mysql: + # [Required] Whether to enable MySQL + enabled: true + # Image settings; default is hub.oepkgs.net/neocopilot/mysql:8-x86 + # Image tags: ["8-x86", "8-arm"] + image: + # Resource limits settings + resourceLimits: {} + # Service settings + service: + # Service type, e.g., NodePort + type: + # When type is NodePort, specify the host port number + nodePort: diff --git a/deploy/en/chart/databases/.helmignore b/deploy/en/chart/databases/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/deploy/en/chart/databases/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/deploy/en/chart/databases/Chart.yaml b/deploy/en/chart/databases/Chart.yaml new file mode 100644 index 000000000..df548c6cd --- /dev/null +++ b/deploy/en/chart/databases/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: euler-copilot-databases +description: Euler Copilot Database Helm Chart +type: application +version: 0.10.0 +appVersion: "0.10.0" diff --git a/deploy/en/chart/databases/templates/NOTES.txt b/deploy/en/chart/databases/templates/NOTES.txt new file mode 100644 index 000000000..e845ceb64 --- /dev/null +++ b/deploy/en/chart/databases/templates/NOTES.txt @@ -0,0 +1,3 @@ +Thank you for using openEuler Intelligence! +The current version is 0.10.0. +The current Chart functionality is: Database Deployment. diff --git a/deploy/en/chart/databases/templates/_helpers.tpl b/deploy/en/chart/databases/templates/_helpers.tpl new file mode 100644 index 000000000..17d02ea44 --- /dev/null +++ b/deploy/en/chart/databases/templates/_helpers.tpl @@ -0,0 +1,15 @@ +{{- define "databases.generateGaussPassword" -}} + {{- /* 使用确定的特殊字符组合 */ -}} + {{- $special := "#!" -}} + + {{- /* 生成基础密码 */ -}} + {{- $base := randAlphaNum 10 -}} + {{- $upper := randAlphaNum 3 | upper -}} + {{- $digits := randNumeric 3 -}} + + {{- /* 组合密码 */ -}} + {{- $password := print $base $upper $digits $special -}} + + {{- /* 转义特殊字符 */ -}} + {{- $password | replace "!" "\\!" | replace "$" "\\$" | replace "&" "\\&" -}} +{{- end -}} diff --git a/deploy/en/chart/databases/templates/minio/minio-storage.yaml b/deploy/en/chart/databases/templates/minio/minio-storage.yaml new file mode 100644 index 000000000..d70fdad49 --- /dev/null +++ b/deploy/en/chart/databases/templates/minio/minio-storage.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.databases.minio.enabled -}} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: minio-storage + namespace: {{ .Release.Namespace }} + annotations: + helm.sh/resource-policy: keep +spec: + storageClassName: {{ default "local-path" .Values.globals.storageClass }} + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ default "10Gi" .Values.storage.minio }} +{{- end -}} \ No newline at end of file diff --git a/deploy/en/chart/databases/templates/minio/minio.yaml b/deploy/en/chart/databases/templates/minio/minio.yaml new file mode 100644 index 000000000..185883b62 --- /dev/null +++ b/deploy/en/chart/databases/templates/minio/minio.yaml @@ -0,0 +1,79 @@ +{{- if .Values.databases.minio.enabled -}} +--- +apiVersion: v1 +kind: Service +metadata: + name: minio-service + namespace: {{ .Release.Namespace }} +spec: + type: {{ default "ClusterIP" .Values.databases.minio.service.type }} + selector: + app: minio + ports: + - port: 9000 + targetPort: 9000 + nodePort: {{ default nil .Values.databases.minio.service.dataNodePort }} + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio-deploy + namespace: {{ .Release.Namespace }} + labels: + app: minio +spec: + replicas: {{ default 1 .Values.globals.replicaCount }} + selector: + matchLabels: + app: minio + template: + metadata: + labels: + app: minio + spec: + automountServiceAccountToken: false + containers: + - name: minio + image: {{ .Values.databases.minio.image | default (printf "%s/neocopilot/minio:empty-%s" (.Values.globals.imageRegistry | default "hub.oepkgs.net") (.Values.globals.arch | default "x86")) }} + imagePullPolicy: {{ default "IfNotPresent" .Values.globals.imagePullPolicy }} + args: + - "server" + - "/data" + - "--console-address" + - ":9001" + ports: + - containerPort: 9000 + protocol: TCP + livenessProbe: + httpGet: + path: /minio/health/live + port: 9000 + scheme: HTTP + failureThreshold: 5 + initialDelaySeconds: 60 + periodSeconds: 90 + env: + - name: TZ + value: "Asia/Shanghai" + - name: MINIO_ROOT_USER + value: minioadmin + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: euler-copilot-database + key: minio-password + volumeMounts: + - mountPath: "/data" + name: minio-data + resources: + requests: + cpu: 0.25 + memory: 256Mi + limits: + {{ toYaml .Values.databases.minio.resourceLimits | nindent 14 }} + volumes: + - name: minio-data + persistentVolumeClaim: + claimName: minio-storage +{{- end -}} diff --git a/deploy/en/chart/databases/templates/mongo/mongo-config.yaml b/deploy/en/chart/databases/templates/mongo/mongo-config.yaml new file mode 100644 index 000000000..9c9594fd3 --- /dev/null +++ b/deploy/en/chart/databases/templates/mongo/mongo-config.yaml @@ -0,0 +1,23 @@ +{{- if .Values.databases.mongo.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: mongo-config + namespace: {{ .Release.Namespace }} +data: + healthcheck.sh: | + #! /bin/bash + + if mongosh --quiet --eval "rs.status().ok" -u ${MONGO_INITDB_ROOT_USERNAME} -p ${MONGO_INITDB_ROOT_PASSWORD} &> /dev/null; then + echo "MongoDB集群状态正常" + exit 0 + else + echo "初始化MongoDB集群" + if ! mongosh --quiet --eval 'rs.initiate({_id: "rs0", members: [{ _id: 0, host: "127.0.0.1:27017" }]});' -u ${MONGO_INITDB_ROOT_USERNAME} -p ${MONGO_INITDB_ROOT_PASSWORD} &> /dev/null; then + echo "初始化MongoDB集群失败！" + exit 1 + fi + echo "初始化MongoDB集群成功！" + exit 0 + fi +{{- end -}} \ No newline at end of file diff --git a/deploy/en/chart/databases/templates/mongo/mongo-storage.yaml b/deploy/en/chart/databases/templates/mongo/mongo-storage.yaml new file mode 100644 index 000000000..db5c313fb --- /dev/null +++ b/deploy/en/chart/databases/templates/mongo/mongo-storage.yaml @@ -0,0 +1,16 @@ +{{- if .Values.databases.mongo.enabled -}} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: mongo-storage + namespace: {{ .Release.Namespace }} + annotations: + helm.sh/resource-policy: keep +spec: + storageClassName: {{ default "local-path" .Values.globals.storageClass }} + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ default "10Gi" .Values.storage.mongo }} +{{- end -}} \ No newline at end of file diff --git a/deploy/en/chart/databases/templates/mongo/mongo.yaml b/deploy/en/chart/databases/templates/mongo/mongo.yaml new file mode 100644 index 000000000..87d750f45 --- /dev/null +++ b/deploy/en/chart/databases/templates/mongo/mongo.yaml @@ -0,0 +1,102 @@ +{{- if .Values.databases.mongo.enabled -}} +--- +apiVersion: v1 +kind: Service +metadata: + name: mongo-db + namespace: {{ .Release.Namespace }} +spec: + type: {{ default "ClusterIP" .Values.databases.mongo.service.type }} + selector: + app: mongo + ports: + - port: 27017 + targetPort: 27017 + nodePort: {{ default nil .Values.databases.mongo.service.nodePort }} + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mongo-deploy + namespace: {{ .Release.Namespace }} + labels: + app: mongo +spec: + replicas: {{ default 1 .Values.globals.replicaCount }} + selector: + matchLabels: + app: mongo + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/mongo/mongo-config.yaml") . | sha256sum }} + labels: + app: mongo + spec: + automountServiceAccountToken: false + containers: + - name: mongo + image: {{ .Values.databases.mongo.image | default (printf "%s/neocopilot/mongo:7.0.16-%s" (.Values.globals.imageRegistry | default "hub.oepkgs.net") (.Values.globals.arch | default "x86")) }} + imagePullPolicy: {{ default "IfNotPresent" .Values.globals.imagePullPolicy }} + command: + - bash + - -c + - | + if [ ! -f "/data/db/file.key" ]; then + openssl rand -base64 756 > /data/db/file.key; + fi + chmod 400 /data/db/file.key; + chown 999:999 /data/db/file.key; + exec docker-entrypoint.sh $$@ + args: + - "mongod" + - "--replSet" + - "rs0" + - "--bind_ip_all" + - "--keyFile" + - "/data/db/file.key" + ports: + - containerPort: 27017 + protocol: TCP + livenessProbe: + exec: + command: + - bash + - /tmp/healthcheck.sh + failureThreshold: 5 + initialDelaySeconds: 30 + periodSeconds: 60 + env: + - name: TZ + value: "Asia/Shanghai" + - name: MONGO_INITDB_ROOT_USERNAME + value: "euler_copilot" + - name: MONGO_INITDB_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: euler-copilot-database + key: mongo-password + - name: MONGO_INITDB_DATABASE + value: euler_copilot + volumeMounts: + - mountPath: /data/db + name: mongo-data + - mountPath: /tmp/healthcheck.sh + name: mongo-init + subPath: healthcheck.sh + resources: + requests: + cpu: 0.25 + memory: 256Mi + limits: + {{ toYaml .Values.databases.mongo.resourceLimits | nindent 14 }} + restartPolicy: Always + volumes: + - name: mongo-data + persistentVolumeClaim: + claimName: mongo-storage + - name: mongo-init + configMap: + name: mongo-config +{{- end -}} diff --git a/deploy/en/chart/databases/templates/opengauss/opengauss-config.yaml b/deploy/en/chart/databases/templates/opengauss/opengauss-config.yaml new file mode 100644 index 000000000..aca7d76ec --- /dev/null +++ b/deploy/en/chart/databases/templates/opengauss/opengauss-config.yaml @@ -0,0 +1,16 @@ +{{- if .Values.databases.opengauss.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: opengauss-config + namespace: {{ .Release.Namespace }} +data: + init.sql: | + CREATE EXTENSION chparser; + CREATE TEXT SEARCH CONFIGURATION chparser (PARSER = chparser); + ALTER TEXT SEARCH CONFIGURATION chparser ADD MAPPING FOR n,v,a,i,e,l WITH simple; + post-init.sh: | + #!/bin/bash + su - omm -c "gs_guc reload -D /var/lib/opengauss/data -c \"behavior_compat_options = 'accept_empty_str'\"" +{{- end -}} + diff --git a/deploy/en/chart/databases/templates/opengauss/opengauss-storage.yaml b/deploy/en/chart/databases/templates/opengauss/opengauss-storage.yaml new file mode 100644 index 000000000..307b464cb --- /dev/null +++ b/deploy/en/chart/databases/templates/opengauss/opengauss-storage.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.databases.opengauss.enabled -}} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: opengauss-storage + namespace: {{ .Release.Namespace }} + annotations: + helm.sh/resource-policy: keep +spec: + storageClassName: {{ default "local-path" .Values.globals.storageClass }} + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ default "10Gi" .Values.storage.opengauss }} +{{- end -}} + diff --git a/deploy/en/chart/databases/templates/opengauss/opengauss.yaml b/deploy/en/chart/databases/templates/opengauss/opengauss.yaml new file mode 100644 index 000000000..215937b20 --- /dev/null +++ b/deploy/en/chart/databases/templates/opengauss/opengauss.yaml @@ -0,0 +1,93 @@ +{{- if .Values.databases.opengauss.enabled -}} +--- +apiVersion: v1 +kind: Service +metadata: + name: opengauss-db + namespace: {{ .Release.Namespace }} +spec: + type: {{ default "ClusterIP" .Values.databases.opengauss.service.type }} + selector: + app: opengauss + ports: + - port: 5432 + targetPort: 5432 + nodePort: {{ default nil .Values.databases.opengauss.service.nodePort }} + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: opengauss-deploy + namespace: {{ .Release.Namespace }} + labels: + app: opengauss +spec: + replicas: {{ default 1 .Values.globals.replicaCount }} + selector: + matchLabels: + app: opengauss + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/opengauss/opengauss-config.yaml") . | sha256sum }} + labels: + app: opengauss + spec: + automountServiceAccountToken: false + containers: + - name: opengauss + image: {{ .Values.databases.opengauss.image | default (printf "%s/neocopilot/opengauss:latest-%s" (.Values.globals.imageRegistry | default "hub.oepkgs.net") (.Values.globals.arch | default "x86")) }} + imagePullPolicy: {{ default "IfNotPresent" .Values.globals.imagePullPolicy }} + ports: + - containerPort: 5432 + protocol: TCP + livenessProbe: + exec: + command: ["/bin/bash", "/docker-entrypoint-initdb.d/post-init.sh"] + failureThreshold: 5 + initialDelaySeconds: 60 + periodSeconds: 90 + env: + - name: TZ + value: "Asia/Shanghai" + - name: GS_DB + value: "postgres" + - name: GS_USERNAME + value: "postgres" + - name: GS_PASSWORD + valueFrom: + secretKeyRef: + name: euler-copilot-database + key: gauss-password + - name: GAUSSLOG + value: /var/log/opengauss + volumeMounts: + - mountPath: /var/lib/opengauss/data + name: opengauss-data + - mountPath: /docker-entrypoint-initdb.d/init.sql + name: opengauss-init + subPath: init.sql + - mountPath: /docker-entrypoint-initdb.d/post-init.sh + name: opengauss-init + subPath: post-init.sh + - mountPath: /var/log/opengauss + name: opengauss-log + resources: + requests: + cpu: 0.5 + memory: 1024Mi + limits: + {{ toYaml .Values.databases.opengauss.resourceLimits | indent 4 | trim }} + volumes: + - name: opengauss-data + persistentVolumeClaim: + claimName: opengauss-storage + - name: opengauss-init + configMap: + name: opengauss-config + defaultMode: 484 + - name: opengauss-log + emptyDir: {} +{{- end -}} + diff --git a/deploy/en/chart/databases/templates/pgsql/pgsql-config.yaml b/deploy/en/chart/databases/templates/pgsql/pgsql-config.yaml new file mode 100644 index 000000000..1f73664cd --- /dev/null +++ b/deploy/en/chart/databases/templates/pgsql/pgsql-config.yaml @@ -0,0 +1,13 @@ +{{- if .Values.databases.pgsql.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: pgsql-config + namespace: {{ .Release.Namespace }} +data: + init.sql: | + CREATE EXTENSION zhparser; + CREATE EXTENSION vector; + CREATE TEXT SEARCH CONFIGURATION zhparser (PARSER = zhparser); + ALTER TEXT SEARCH CONFIGURATION zhparser ADD MAPPING FOR n,v,a,i,e,l WITH simple; +{{- end -}} \ No newline at end of file diff --git a/deploy/en/chart/databases/templates/pgsql/pgsql-storage.yaml b/deploy/en/chart/databases/templates/pgsql/pgsql-storage.yaml new file mode 100644 index 000000000..5b5a16a31 --- /dev/null +++ b/deploy/en/chart/databases/templates/pgsql/pgsql-storage.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.databases.pgsql.enabled -}} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pgsql-storage + namespace: {{ .Release.Namespace }} + annotations: + helm.sh/resource-policy: keep +spec: + storageClassName: {{ default "local-path" .Values.globals.storageClass }} + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ default "10Gi" .Values.storage.pgsql }} +{{- end -}} \ No newline at end of file diff --git a/deploy/en/chart/databases/templates/pgsql/pgsql.yaml b/deploy/en/chart/databases/templates/pgsql/pgsql.yaml new file mode 100644 index 000000000..97b92bc8e --- /dev/null +++ b/deploy/en/chart/databases/templates/pgsql/pgsql.yaml @@ -0,0 +1,84 @@ +{{- if .Values.databases.pgsql.enabled -}} +--- +apiVersion: v1 +kind: Service +metadata: + name: opengauss-db + namespace: {{ .Release.Namespace }} +spec: + type: {{ default "ClusterIP" .Values.databases.pgsql.service.type }} + selector: + app: pgsql + ports: + - port: 5432 + targetPort: 5432 + nodePort: {{ default nil .Values.databases.pgsql.service.nodePort }} + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pgsql-deploy + namespace: {{ .Release.Namespace }} + labels: + app: pgsql +spec: + replicas: {{ default 1 .Values.globals.replicaCount }} + selector: + matchLabels: + app: pgsql + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/pgsql/pgsql-config.yaml") . | sha256sum }} + labels: + app: pgsql + spec: + automountServiceAccountToken: false + containers: + - name: pgsql + image: {{ .Values.databases.pgsql.image | default (printf "%s/neocopilot/pgsql-empty:pg16-%s" (.Values.globals.imageRegistry | default "hub.oepkgs.net") (.Values.globals.arch | default "x86")) }} + imagePullPolicy: {{ default "IfNotPresent" .Values.globals.imagePullPolicy }} + ports: + - containerPort: 5432 + protocol: TCP + livenessProbe: + exec: + command: + - pg_isready + - -d postgres -U postgres + failureThreshold: 5 + initialDelaySeconds: 60 + periodSeconds: 90 + env: + - name: TZ + value: "Asia/Shanghai" + - name: POSTGRES_DB + value: "postgres" + - name: POSTGRES_USER + value: "postgres" + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: euler-copilot-database + key: gauss-password + volumeMounts: + - mountPath: /var/lib/postgresql/data + name: pgsql-data + - mountPath: /docker-entrypoint-initdb.d/init.sql + name: pgsql-init + subPath: init.sql + resources: + requests: + cpu: 0.25 + memory: 512Mi + limits: + {{ toYaml .Values.databases.pgsql.resourceLimits | nindent 14 }} + volumes: + - name: pgsql-data + persistentVolumeClaim: + claimName: pgsql-storage + - name: pgsql-init + configMap: + name: pgsql-config +{{- end -}} \ No newline at end of file diff --git a/deploy/en/chart/databases/templates/redis/redis.yaml b/deploy/en/chart/databases/templates/redis/redis.yaml new file mode 100644 index 000000000..2e3811380 --- /dev/null +++ b/deploy/en/chart/databases/templates/redis/redis.yaml @@ -0,0 +1,77 @@ +{{- if .Values.databases.redis.enabled -}} +--- +apiVersion: v1 +kind: Service +metadata: + name: redis-db + namespace: {{ .Release.Namespace }} +spec: + type: {{ default "ClusterIP" .Values.databases.redis.service.type }} + selector: + app: redis + ports: + - port: 6379 + targetPort: 6379 + nodePort: {{ default nil .Values.databases.redis.service.nodePort }} + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis-deploy + namespace: {{ .Release.Namespace }} + labels: + app: redis +spec: + replicas: {{ default 1 .Values.globals.replicaCount }} + selector: + matchLabels: + app: redis + template: + metadata: + labels: + app: redis + spec: + automountServiceAccountToken: false + containers: + - name: redis + image: {{ .Values.databases.redis.image | default (printf "%s/neocopilot/redis:7.4-alpine-%s" (.Values.globals.imageRegistry | default "hub.oepkgs.net") (.Values.globals.arch | default "x86")) }} + imagePullPolicy: {{ default "IfNotPresent" .Values.globals.imagePullPolicy }} + command: + - redis-server + - --requirepass $(REDIS_PASSWORD) + ports: + - containerPort: 6379 + protocol: TCP + livenessProbe: + exec: + command: + - sh + - -c + - redis-cli -a $REDIS_PASSWORD ping + failureThreshold: 5 + initialDelaySeconds: 60 + periodSeconds: 90 + env: + - name: TZ + value: "Asia/Shanghai" + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: euler-copilot-database + key: redis-password + volumeMounts: + - mountPath: /tmp + name: redis-tmp + resources: + requests: + cpu: 0.1 + memory: 64Mi + limits: + {{ toYaml .Values.databases.redis.resourceLimits | nindent 14 }} + restartPolicy: Always + volumes: + - name: redis-tmp + emptyDir: + medium: Memory +{{- end -}} diff --git a/deploy/en/chart/databases/templates/secrets.yaml b/deploy/en/chart/databases/templates/secrets.yaml new file mode 100644 index 000000000..b146644ca --- /dev/null +++ b/deploy/en/chart/databases/templates/secrets.yaml @@ -0,0 +1,30 @@ +{{- $databaseSecret := (lookup "v1" "Secret" .Release.Namespace "euler-copilot-database") -}} +{{- if $databaseSecret -}} +apiVersion: v1 +kind: Secret +metadata: + name: euler-copilot-database + namespace: {{ .Release.Namespace }} + annotations: + helm.sh/resource-policy: keep +type: Opaque +stringData: + redis-password: {{ index $databaseSecret.data "redis-password" | b64dec | quote }} + mongo-password: {{ index $databaseSecret.data "mongo-password" | b64dec | quote }} + minio-password: {{ index $databaseSecret.data "minio-password" | b64dec | quote }} + gauss-password: {{ index $databaseSecret.data "gauss-password" | b64dec | quote }} +{{- else -}} +apiVersion: v1 +kind: Secret +metadata: + name: euler-copilot-database + namespace: {{ .Release.Namespace }} + annotations: + helm.sh/resource-policy: keep +type: Opaque +stringData: + redis-password: {{ randAlphaNum 20 | quote }} + mongo-password: {{ randAlphaNum 20 | quote }} + minio-password: {{ randAlphaNum 20 | quote }} + gauss-password: {{ include "databases.generateGaussPassword" . | quote }} +{{- end -}} diff --git a/deploy/en/chart/databases/values.yaml b/deploy/en/chart/databases/values.yaml new file mode 100644 index 000000000..76d49d22d --- /dev/null +++ b/deploy/en/chart/databases/values.yaml @@ -0,0 +1,106 @@ +globals: + # Node Architecture: default is x86 + # Node settings: ["x86", "arm"] + arch: + # Deployment replica count, default is 1 + replicaCount: + # Image pull policy, default is IfNotPresent + imagePullPolicy: + # Storage class, default is local-path + storageClass: + +# Storage Settings +storage: + # MinIO storage size, default is 10GB + minio: + # MongoDB storage size, default is 10GB + mongo: + # Vector database storage size, default is 10GB + opengauss: + # PostgreSQL storage size, default is 10GB + pgsql: + +# Domain Settings +domain: + # Should be modified to the domain name bound to MinIO Console. + minioConsole: + +databases: + minio: + # [Required] Whether to deploy MinIO instance + enabled: true + # Image settings: default is hub.oepkgs.net/neocopilot/minio:empty-x86 + # Image versions: ["empty-x86", "empty-arm"] + image: + # Resource limits settings + resourceLimits: {} + # Service settings + service: + # Service type, e.g., NodePort + type: + # When type is NodePort, specify the host port number corresponding to MinIO data port + dataNodePort: + # When type is NodePort, specify the host port number corresponding to MinIO console + consoleNodePort: + # Ingress settings + ingress: + # [Required] Whether to expose MinIO Console + enabled: true + # Ingress URL prefix + prefix: / + mongo: + # [Required] Whether to deploy MongoDB database instance + enabled: true + # Image settings; default is hub.oepkgs.net/neocopilot/mongo:7.0.16-x86 + # Image versions: ["7.0.16-x86", "7.0.16-arm"] + image: + # Resource limits settings + resourceLimits: {} + # Service settings + service: + # [Required] Service type, e.g., NodePort + type: + # When type is nodePort, specify the host port number + nodePort: + redis: + # [Required] Whether to deploy Redis instance + enabled: true + # Image settings, default is hub.oepkgs.net/neocopilot/redis:7.4-alpine-x86 + # Image versions: ["7.4-alpine-x86", "7.4-alpine-arm"] + image: + # Resource limits settings + resourceLimits: {} + # Service settings + service: + # Service type, e.g., NodePort + type: + # When type is nodePort, specify the host port number + nodePort: + opengauss: + # [Required] Whether to deploy OpenGauss instance + enabled: true + # Image settings, default is hub.oepkgs.net/neocopilot/opengauss:latest-x86 + # Image versions: ["latest-x86", "latest-arm"] + image: + # Resource limits settings + resourceLimits: {} + # Service settings + service: + # Service type, e.g., NodePort + type: + # When type is NodePort, specify the host port number + nodePort: + pgsql: + # [Required] Whether to deploy PostgreSQL instance + enabled: false + # Image settings, default is hub.oepkgs.net/neocopilot/pgsql-empty:pg16-x86 + # Image versions: ["pg16-x86", "pg16-arm"] + image: + # Resource limits settings + resourceLimits: {} + # Service settings + service: + # Service type, e.g., NodePort + type: + # When type is NodePort, specify the host port number + nodePort: diff --git a/deploy/en/chart/euler_copilot/.helmignore b/deploy/en/chart/euler_copilot/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/deploy/en/chart/euler_copilot/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/deploy/en/chart/euler_copilot/Chart.yaml b/deploy/en/chart/euler_copilot/Chart.yaml new file mode 100644 index 000000000..781b5d466 --- /dev/null +++ b/deploy/en/chart/euler_copilot/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: euler-copilot +description: Euler Copilot Helm Chart +type: application +version: 0.10.0 +appVersion: "0.10.0" diff --git a/deploy/en/chart/euler_copilot/configs/framework/config.toml b/deploy/en/chart/euler_copilot/configs/framework/config.toml new file mode 100644 index 000000000..64a1a6a03 --- /dev/null +++ b/deploy/en/chart/euler_copilot/configs/framework/config.toml @@ -0,0 +1,66 @@ +[deploy] +mode = 'local' +cookie = 'domain' +data_dir = '/app/data' + +[login] +provider = 'authhub' +[login.settings] +host = '{{ .Values.domain.authhub | default "http://127.0.0.1:30081" }}' +host_inner = 'http://authhub-backend-service.{{ .Release.Namespace }}.svc.cluster.local:11120' +login_api = '{{ .Values.domain.euler_copilot | default "http://127.0.0.1:30080" }}/api/auth/login' +app_id = '${clientId}' +app_secret = '${clientSecret}' + +[fastapi] +domain = '{{ regexFind "^[^:]+" (.Values.domain.euler_copilot | default "http://127.0.0.1:30080" | replace "http://" "" | replace "https://" "") }}' + +[security] +half_key1 = '${halfKey1}' +half_key2 = '${halfKey2}' +half_key3 = '${halfKey3}' +jwt_key = '${jwtKey}' + +[embedding] +type = '{{ default "openai" .Values.models.embedding.type }}' +endpoint = '{{ .Values.models.embedding.endpoint }}' +api_key = '{{ .Values.models.embedding.key }}' +model = '{{ default "bge-m3" .Values.models.embedding.name }}' + +[rag] +rag_service = 'http://rag-service.{{ .Release.Namespace }}.svc.cluster.local:9988' + +[mongodb] +host = 'mongo-db.{{ .Release.Namespace }}.svc.cluster.local' +port = 27017 +user = 'euler_copilot' +password = '${mongo-password}' +database = 'euler_copilot' + +[minio] +endpoint = 'minio-service.{{ .Release.Namespace }}.svc.cluster.local:9000' +access_key = 'minioadmin' +secret_key = '${minio-password}' +secure = false + +[llm] +endpoint = '{{ .Values.models.answer.endpoint }}' +key = '{{ .Values.models.answer.key }}' +model = '{{ .Values.models.answer.name }}' +max_tokens = {{ default 8192 .Values.models.answer.maxTokens }} +temperature = {{ default 0.7 .Values.models.answer.temperature }} + +[function_call] +backend = '{{ default "ollama" .Values.models.functionCall.backend }}' +endpoint = '{{ default .Values.models.answer.endpoint .Values.models.functionCall.endpoint }}' +model = '{{ default .Values.models.answer.name .Values.models.functionCall.name }}' +api_key = '{{ default .Values.models.answer.key .Values.models.functionCall.key }}' +max_tokens = {{ default .Values.models.answer.maxTokens .Values.models.functionCall.maxTokens }} +temperature = {{ default 0.7 .Values.models.functionCall.temperature }} + +[check] +enable = false +words_list = "" + +[extra] +sql_url = '' diff --git a/deploy/en/chart/euler_copilot/configs/rag/.env b/deploy/en/chart/euler_copilot/configs/rag/.env new file mode 100644 index 000000000..0708fd076 --- /dev/null +++ b/deploy/en/chart/euler_copilot/configs/rag/.env @@ -0,0 +1,60 @@ +# Fastapi +UVICORN_IP=0.0.0.0 +UVICORN_PORT=9988 +SSL_CERTFILE= +SSL_KEYFILE= +SSL_ENABLE=false +LOG_METHOD=stdout + +# opengauss +DATABASE_TYPE=opengauss +DATABASE_HOST=opengauss-db.{{ .Release.Namespace }}.svc.cluster.local +DATABASE_PORT=5432 +DATABASE_USER=postgres +DATABASE_PASSWORD=${gauss-password} +DATABASE_DB=postgres + +# MinIO +MINIO_ENDPOINT=minio-service.{{ .Release.Namespace }}.svc.cluster.local:9000 +MINIO_ACCESS_KEY=minioadmin +MINIO_SECRET_KEY=${minio-password} +MINIO_SECURE=false + +# MongoDB +MONGODB_USER = euler_copilot +MONGODB_PASSWORD = ${mongo-password} +MONGODB_HOST = mongo-db.{{ .Release.Namespace }}.svc.cluster.local +MONGODB_PORT = 27017 +MONGODB_DATABASE = euler_copilot + +# Task +TASK_RETRY_TIME=3 + +# Embedding +EMBEDDING_TYPE={{ default "openai" .Values.models.embedding.type }} +EMBEDDING_ENDPOINT={{ .Values.models.embedding.endpoint }}/embeddings +EMBEDDING_API_KEY={{ .Values.models.embedding.key }} +EMBEDDING_MODEL_NAME={{ .Values.models.embedding.name }} + +# Token +CSRF_KEY=${csrfKey} +SESSION_TTL=1440 + +# PROMPT_PATH +PROMPT_PATH=/rag-service/data_chain/common/prompt.yaml +# Stop Words PATH +STOP_WORDS_PATH=/rag-service/data_chain/common/stopwords.txt + +#Security +HALF_KEY1=${halfKey1} +HALF_KEY2=${halfKey2} +HALF_KEY3=${halfKey3} + +#LLM config +MODEL_NAME={{ .Values.models.answer.name }} +OPENAI_API_BASE={{ .Values.models.answer.endpoint }} +OPENAI_API_KEY={{ default "" .Values.models.answer.key }} +MAX_TOKENS={{ default 2048 .Values.models.answer.maxTokens }} + +# DOCUMENT PARSER +DOCUMENT_PARSE_USE_CPU_LIMIT = 4 diff --git a/deploy/en/chart/euler_copilot/configs/rag/.env-sql b/deploy/en/chart/euler_copilot/configs/rag/.env-sql new file mode 100644 index 000000000..5553f277a --- /dev/null +++ b/deploy/en/chart/euler_copilot/configs/rag/.env-sql @@ -0,0 +1,29 @@ +# FastAPI +UVICORN_IP=0.0.0.0 +UVICORN_PORT=9015 + +# Postgres +DATABASE_TYPE=opengauss +DATABASE_HOST=opengauss-db.{{ .Release.Namespace }}.svc.cluster.local +DATABASE_PORT=5432 +DATABASE_USER=postgres +DATABASE_PASSWORD=${gauss-password} +DATABASE_DB=postgres + +# QWEN +LLM_MODEL={{ .Values.models.answer.name }} +LLM_URL={{ .Values.models.answer.endpoint }}/v1 +LLM_KEY={{ default "" .Values.models.answer.key }} +LLM_MAX_TOKENS={{ default 2048 .Values.models.answer.maxTokens }} + + +# Embedding +EMBEDDING_TYPE={{ default "openai" .Values.models.embedding.type }} +EMBEDDING_ENDPOINT={{ .Values.models.embedding.endpoint }}/embeddings +EMBEDDING_API_KEY={{ .Values.models.embedding.key }} +EMBEDDING_MODEL_NAME={{ .Values.models.embedding.name }} + +# security +HALF_KEY1=${halfKey1} +HALF_KEY2=${halfKey2} +HALF_KEY3=${halfKey3} diff --git a/deploy/en/chart/euler_copilot/templates/NOTES.txt b/deploy/en/chart/euler_copilot/templates/NOTES.txt new file mode 100644 index 000000000..48a494921 --- /dev/null +++ b/deploy/en/chart/euler_copilot/templates/NOTES.txt @@ -0,0 +1,5 @@ +Thank you for using openEuler Intelligence! +Current version is 0.10.0. +The current Chart functionality is: openEuler Intelligence Core Components Deployment. + +For more project updates and sharing sessions, please follow openEuler sig-intelligence: https://www.openeuler.org/en/sig/sig-detail/?name=sig-intelligence diff --git a/deploy/en/chart/euler_copilot/templates/framework/framework-config.yaml b/deploy/en/chart/euler_copilot/templates/framework/framework-config.yaml new file mode 100644 index 000000000..328803d4f --- /dev/null +++ b/deploy/en/chart/euler_copilot/templates/framework/framework-config.yaml @@ -0,0 +1,21 @@ +{{- if .Values.euler_copilot.framework.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: framework-config + namespace: {{ .Release.Namespace }} +data: + config.toml: |- +{{ tpl (.Files.Get "configs/framework/config.toml") . | indent 4 }} + copy-config.yaml: |- + copy: + - from: /config/config.toml + to: /config-rw/config.toml + mode: + uid: 0 + gid: 0 + mode: "0o650" + secrets: + - /db-secrets + - /system-secrets +{{- end -}} \ No newline at end of file diff --git a/deploy/en/chart/euler_copilot/templates/framework/framework-storage.yaml b/deploy/en/chart/euler_copilot/templates/framework/framework-storage.yaml new file mode 100644 index 000000000..25047e1b8 --- /dev/null +++ b/deploy/en/chart/euler_copilot/templates/framework/framework-storage.yaml @@ -0,0 +1,31 @@ +{{- if .Values.euler_copilot.framework.enabled -}} +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: framework-semantics + namespace: {{ .Release.Namespace }} +spec: + storageClassName: {{ default "local-path" .Values.globals.storageClass }} + capacity: + storage: {{ default "10Gi" .Values.storage.frameworkSemantics.size }} + accessModes: + - ReadWriteOnce + hostPath: + path: {{ default "/var/lib/eulercopilot" .Values.storage.frameworkSemantics.path }} + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: framework-semantics-claim + namespace: {{ .Release.Namespace }} +spec: + storageClassName: {{ default "local-path" .Values.globals.storageClass }} + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ default "5Gi" .Values.storage.frameworkSemantics.size }} + volumeName: framework-semantics +{{- end -}} diff --git a/deploy/en/chart/euler_copilot/templates/framework/framework.yaml b/deploy/en/chart/euler_copilot/templates/framework/framework.yaml new file mode 100644 index 000000000..cc94cdccd --- /dev/null +++ b/deploy/en/chart/euler_copilot/templates/framework/framework.yaml @@ -0,0 +1,120 @@ +{{- if .Values.euler_copilot.framework.enabled -}} +--- +apiVersion: v1 +kind: Service +metadata: + name: framework-service + namespace: {{ .Release.Namespace }} +spec: + type: {{ default "ClusterIP" .Values.euler_copilot.framework.service.type }} + selector: + app: framework + ports: + - name: framework + port: 8002 + targetPort: 8002 + nodePort: {{ default nil .Values.euler_copilot.framework.service.nodePort }} + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: framework-deploy + namespace: {{ .Release.Namespace }} + labels: + app: framework +spec: + selector: + matchLabels: + app: framework + template: + metadata: + annotations: + checksum/secret: {{ include (print $.Template.BasePath "/framework/framework-config.yaml") . | sha256sum }} + labels: + app: framework + spec: + automountServiceAccountToken: false + containers: + - name: framework + image: {{ .Values.euler_copilot.framework.image | default (printf "%s/neocopilot/euler-copilot-framework:0.10.0-%s" (.Values.globals.imageRegistry | default "hub.oepkgs.net") (.Values.globals.arch | default "x86")) }} + imagePullPolicy: {{ default "IfNotPresent" .Values.globals.imagePullPolicy }} + ports: + - containerPort: 8002 + protocol: TCP + livenessProbe: + httpGet: + path: /health_check + port: 8002 + scheme: HTTP + failureThreshold: 5 + initialDelaySeconds: 60 + periodSeconds: 90 + env: + - name: TZ + value: "Asia/Shanghai" + - name: CONFIG + value: "/app/config/config.toml" + volumeMounts: + - mountPath: /app/config + name: framework-shared + - mountPath: /tmp + name: framework-tmp-volume + - mountPath: /app/static + name: web-static + - mountPath: /app/data + name: framework-semantics-vl + securityContext: + readOnlyRootFilesystem: {{ default false .Values.euler_copilot.framework.readOnly }} + resources: + requests: + cpu: 0.2 + memory: 512Mi + limits: + {{ toYaml .Values.euler_copilot.framework.resourceLimits | nindent 14 }} + initContainers: + - name: framework-copy + image: {{ .Values.euler_copilot.secretInject.image | default (printf "%s/neocopilot/secret_inject:dev-%s" (.Values.globals.imageRegistry | default "hub.oepkgs.net") (.Values.globals.arch | default "x86")) }} + imagePullPolicy: {{ default "IfNotPresent" .Values.globals.imagePullPolicy }} + command: + - python3 + - ./main.py + - --config + - config.yaml + - --copy + volumeMounts: + - mountPath: /config/config.toml + name: framework-config + subPath: config.toml + - mountPath: /app/config.yaml + name: framework-config + subPath: copy-config.yaml + - mountPath: /config-rw + name: framework-shared + - mountPath: /db-secrets + name: database-secrets + - mountPath: /system-secrets + name: system-secrets + volumes: + - name: framework-config + configMap: + name: framework-config + - name: framework-semantics-vl + persistentVolumeClaim: + claimName: framework-semantics-claim + - name: database-secrets + secret: + secretName: euler-copilot-database + - name: system-secrets + secret: + secretName: euler-copilot-system + - name: web-static + persistentVolumeClaim: + claimName: web-static + - name: framework-tmp-volume + emptyDir: + medium: Memory + - name: framework-shared + emptyDir: + medium: Memory +{{- end -}} diff --git a/deploy/en/chart/euler_copilot/templates/rag-web/rag-web-config.yaml b/deploy/en/chart/euler_copilot/templates/rag-web/rag-web-config.yaml new file mode 100644 index 000000000..cc7ce1ba7 --- /dev/null +++ b/deploy/en/chart/euler_copilot/templates/rag-web/rag-web-config.yaml @@ -0,0 +1,10 @@ +{{- if .Values.euler_copilot.rag_web.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: rag-web-config + namespace: {{ .Release.Namespace }} +data: + .env: |- + DATA_CHAIN_BACEND_URL=http://rag-service.{{ .Release.Namespace }}.svc.cluster.local:9988 +{{- end -}} diff --git a/deploy/en/chart/euler_copilot/templates/rag-web/rag-web.yaml b/deploy/en/chart/euler_copilot/templates/rag-web/rag-web.yaml new file mode 100644 index 000000000..2b0ee2b83 --- /dev/null +++ b/deploy/en/chart/euler_copilot/templates/rag-web/rag-web.yaml @@ -0,0 +1,80 @@ +{{- if .Values.euler_copilot.rag_web.enabled -}} +--- +apiVersion: v1 +kind: Service +metadata: + name: rag-web-service + namespace: {{ .Release.Namespace }} +spec: + type: {{ default "ClusterIP" .Values.euler_copilot.rag_web.service.type }} + selector: + app: rag-web + ports: + - port: 9888 + targetPort: 9888 + nodePort: {{ default nil .Values.euler_copilot.rag_web.service.nodePort }} + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rag-web-deploy + namespace: {{ .Release.Namespace }} + labels: + app: rag-web +spec: + replicas: {{ default 1 .Values.globals.replicaCount }} + selector: + matchLabels: + app: rag-web + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/rag-web/rag-web-config.yaml") . | sha256sum }} + labels: + app: rag-web + spec: + automountServiceAccountToken: false + containers: + - name: rag-web + image: {{ .Values.euler_copilot.rag_web.image | default (printf "%s/neocopilot/data_chain_web:0.10.0-%s" (.Values.globals.imageRegistry | default "hub.oepkgs.net") (.Values.globals.arch | default "x86")) }} + imagePullPolicy: {{ default "IfNotPresent" .Values.globals.imagePullPolicy }} + ports: + - containerPort: 9888 + protocol: TCP + livenessProbe: + httpGet: + path: / + port: 9888 + scheme: HTTP + failureThreshold: 5 + initialDelaySeconds: 60 + periodSeconds: 90 + env: + - name: TZ + value: "Asia/Shanghai" + volumeMounts: + - mountPath: /config + name: rag-web-config-volume + - mountPath: /var/lib/nginx/tmp + name: rag-web-tmp + - mountPath: /opt/.env + name: rag-web-env-volume + subPath: .env + resources: + requests: + cpu: 0.05 + memory: 64Mi + limits: + {{ toYaml .Values.euler_copilot.rag_web.resourceLimits | nindent 14 }} + volumes: + - name: rag-web-config-volume + emptyDir: + medium: Memory + - name: rag-web-env-volume + configMap: + name: rag-web-config + - name: rag-web-tmp + emptyDir: + medium: Memory +{{- end -}} diff --git a/deploy/en/chart/euler_copilot/templates/rag/rag-config.yaml b/deploy/en/chart/euler_copilot/templates/rag/rag-config.yaml new file mode 100644 index 000000000..6fb273af1 --- /dev/null +++ b/deploy/en/chart/euler_copilot/templates/rag/rag-config.yaml @@ -0,0 +1,32 @@ +{{- if .Values.euler_copilot.rag.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: rag-config + namespace: {{ .Release.Namespace }} +data: + .env: |- +{{ tpl (.Files.Get "configs/rag/.env") . | indent 4}} + .env-sql: |- +{{ tpl (.Files.Get "configs/rag/.env-sql") . | indent 4}} + copy-config.yaml: |- + copy: + - from: /config/.env + to: /config-rw/.env + mode: + uid: 0 + gid: 0 + mode: "0o650" + secrets: + - /db-secrets + - /system-secrets + - from: /config/.env-sql + to: /config-rw/.env-sql + mode: + uid: 0 + gid: 0 + mode: "0o650" + secrets: + - /db-secrets + - /system-secrets +{{- end -}} diff --git a/deploy/en/chart/euler_copilot/templates/rag/rag.yaml b/deploy/en/chart/euler_copilot/templates/rag/rag.yaml new file mode 100644 index 000000000..334ec6d73 --- /dev/null +++ b/deploy/en/chart/euler_copilot/templates/rag/rag.yaml @@ -0,0 +1,112 @@ +{{- if .Values.euler_copilot.rag.enabled -}} +--- +apiVersion: v1 +kind: Service +metadata: + name: rag-service + namespace: {{ .Release.Namespace }} +spec: + type: {{ default "ClusterIP" .Values.euler_copilot.rag.service.type }} + selector: + app: rag + ports: + - name: rag + port: 9988 + targetPort: 9988 + nodePort: {{ default nil .Values.euler_copilot.rag.service.nodePort }} + - name: rag-sql + port: 9015 + targetPort: 9015 + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rag-deploy + namespace: {{ .Release.Namespace }} + labels: + app: rag +spec: + replicas: {{ default 1 .Values.globals.replicaCount }} + selector: + matchLabels: + app: rag + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/rag/rag-config.yaml") . | sha256sum }} + labels: + app: rag + spec: + automountServiceAccountToken: false + containers: + - name: rag + image: {{ .Values.euler_copilot.rag.image | default (printf "%s/neocopilot/data_chain_back_end:0.10.0-%s" (.Values.globals.imageRegistry | default "hub.oepkgs.net") (.Values.globals.arch | default "x86")) }} + imagePullPolicy: {{ default "IfNotPresent" .Values.globals.imagePullPolicy }} + ports: + - containerPort: 9988 + protocol: TCP + livenessProbe: + httpGet: + path: /health_check + port: 9988 + scheme: HTTP + failureThreshold: 5 + initialDelaySeconds: 60 + periodSeconds: 90 + env: + - name: TZ + value: "Asia/Shanghai" + volumeMounts: + - mountPath: /rag-service/data_chain/common/.env + name: rag-shared + subPath: .env + - mountPath: /rag-service/chat2db/common/.env + name: rag-shared + subPath: .env-sql + resources: + requests: + cpu: 0.25 + memory: 512Mi + limits: + {{ toYaml .Values.euler_copilot.rag.resourceLimits | nindent 14 }} + initContainers: + - name: rag-copy-secret + image: {{ .Values.euler_copilot.secretInject.image | default (printf "%s/neocopilot/secret_inject:dev-%s" (.Values.globals.imageRegistry | default "hub.oepkgs.net") (.Values.globals.arch | default "x86")) }} + imagePullPolicy: {{ default "IfNotPresent" .Values.globals.imagePullPolicy }} + command: + - python3 + - ./main.py + - --config + - config.yaml + - --copy + volumeMounts: + - mountPath: /config/.env + name: rag-config-vl + subPath: .env + - mountPath: /config/.env-sql + name: rag-config-vl + subPath: .env-sql + - mountPath: /app/config.yaml + name: rag-config-vl + subPath: copy-config.yaml + - mountPath: /config-rw + name: rag-shared + - mountPath: /db-secrets + name: database-secret + - mountPath: /system-secrets + name: system-secret + volumes: + - name: rag-config-vl + configMap: + name: rag-config + - name: database-secret + secret: + secretName: euler-copilot-database + - name: system-secret + secret: + secretName: euler-copilot-system + - name: rag-shared + emptyDir: + medium: Memory +{{- end -}} diff --git a/deploy/en/chart/euler_copilot/templates/secrets.yaml b/deploy/en/chart/euler_copilot/templates/secrets.yaml new file mode 100644 index 000000000..5e7faa723 --- /dev/null +++ b/deploy/en/chart/euler_copilot/templates/secrets.yaml @@ -0,0 +1,36 @@ +{{- $systemSecret := (lookup "v1" "Secret" .Release.Namespace "euler-copilot-system") -}} +{{- if $systemSecret -}} +apiVersion: v1 +kind: Secret +metadata: + name: euler-copilot-system + namespace: {{ .Release.Namespace }} + annotations: + helm.sh/resource-policy: keep +type: Opaque +stringData: + jwtKey: {{ index $systemSecret.data.jwtKey | b64dec }} + halfKey1: {{ index $systemSecret.data.halfKey1 | b64dec }} + halfKey2: {{ index $systemSecret.data.halfKey2 | b64dec }} + halfKey3: {{ index $systemSecret.data.halfKey3 | b64dec }} + csrfKey: {{ index $systemSecret.data.csrfKey | b64dec }} + clientId: {{ .Values.login.client.id }} + clientSecret: {{ .Values.login.client.secret }} +{{- else -}} +apiVersion: v1 +kind: Secret +metadata: + name: euler-copilot-system + namespace: {{ .Release.Namespace }} + annotations: + helm.sh/resource-policy: keep +type: Opaque +stringData: + jwtKey: {{ randAlphaNum 32 | b64enc }} + halfKey1: {{ randAlphaNum 32 }} + halfKey2: {{ randAlphaNum 32 }} + halfKey3: {{ randAlphaNum 32 }} + csrfKey: {{ randAlphaNum 32 | b64enc}} + clientId: {{ .Values.login.client.id }} + clientSecret: {{ .Values.login.client.secret }} +{{- end -}} diff --git a/deploy/en/chart/euler_copilot/templates/web/web-config.yaml b/deploy/en/chart/euler_copilot/templates/web/web-config.yaml new file mode 100644 index 000000000..1793023e1 --- /dev/null +++ b/deploy/en/chart/euler_copilot/templates/web/web-config.yaml @@ -0,0 +1,11 @@ +{{- if .Values.euler_copilot.web.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: web-config + namespace: {{ .Release.Namespace }} +data: + .env: |- + RAG_WEB_URL=http://rag-web-service.{{ .Release.Namespace }}.svc.cluster.local:9888 + FRAMEWORK_URL=http://framework-service.{{ .Release.Namespace }}.svc.cluster.local:8002 +{{- end -}} \ No newline at end of file diff --git a/deploy/en/chart/euler_copilot/templates/web/web-storage.yaml b/deploy/en/chart/euler_copilot/templates/web/web-storage.yaml new file mode 100644 index 000000000..a3d7c11e4 --- /dev/null +++ b/deploy/en/chart/euler_copilot/templates/web/web-storage.yaml @@ -0,0 +1,15 @@ +{{- if .Values.euler_copilot.web.enabled -}} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: web-static + namespace: {{ .Release.Namespace }} +spec: + storageClassName: {{ default "local-path" .Values.globals.storageClass }} + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ default "10Gi" .Values.storage.webAsset.size }} + volumeName: web-static +{{- end -}} \ No newline at end of file diff --git a/deploy/en/chart/euler_copilot/templates/web/web.yaml b/deploy/en/chart/euler_copilot/templates/web/web.yaml new file mode 100644 index 000000000..3a027427d --- /dev/null +++ b/deploy/en/chart/euler_copilot/templates/web/web.yaml @@ -0,0 +1,87 @@ +{{- if .Values.euler_copilot.web.enabled -}} +--- +apiVersion: v1 +kind: Service +metadata: + name: web-service + namespace: {{ .Release.Namespace }} +spec: + type: {{ default "NodePort" .Values.euler_copilot.web.service.type }} + selector: + app: web + ports: + - port: 8080 + targetPort: 8080 + nodePort: {{ default 30080 .Values.euler_copilot.web.service.nodePort }} + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: web-deploy + namespace: {{ .Release.Namespace }} + labels: + app: web +spec: + selector: + matchLabels: + app: web + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/web/web-config.yaml") . | sha256sum }} + labels: + app: web + spec: + automountServiceAccountToken: false + containers: + - name: web + image: {{ .Values.euler_copilot.web.image | default (printf "%s/neocopilot/euler-copilot-web:0.10.0-%s" (.Values.globals.imageRegistry | default "hub.oepkgs.net") (.Values.globals.arch | default "x86")) }} + imagePullPolicy: {{ default "IfNotPresent" .Values.globals.imagePullPolicy }} + ports: + - containerPort: 8080 + protocol: TCP + livenessProbe: + httpGet: + path: / + port: 8080 + scheme: HTTP + failureThreshold: 5 + initialDelaySeconds: 60 + periodSeconds: 90 + env: + - name: TZ + value: "Asia/Shanghai" + volumeMounts: + - mountPath: /config + name: web-config-volume + - mountPath: /var/lib/nginx/tmp + name: web-tmp + - mountPath: /opt/.env + name: web-env-volume + subPath: .env + - mountPath: /usr/share/nginx/html/static + name: web-static + resources: + requests: + cpu: 0.05 + memory: 64Mi + limits: + {{ toYaml .Values.euler_copilot.web.resourceLimits | nindent 14 }} + securityContext: + readOnlyRootFilesystem: {{ default false .Values.euler_copilot.web.readOnly }} + restartPolicy: Always + volumes: + - name: web-static + persistentVolumeClaim: + claimName: web-static + - name: web-config-volume + emptyDir: + medium: Memory + - name: web-env-volume + configMap: + name: web-config + - name: web-tmp + emptyDir: + medium: Memory +{{- end -}} diff --git a/deploy/en/chart/euler_copilot/values.yaml b/deploy/en/chart/euler_copilot/values.yaml new file mode 100644 index 000000000..a16527e75 --- /dev/null +++ b/deploy/en/chart/euler_copilot/values.yaml @@ -0,0 +1,155 @@ +# Global Settings +globals: + # Node Architecture: default is x86 + # [Required] Node settings: ["x86", "arm"] + arch: + # Image pull policy, default is IfNotPresent + imagePullPolicy: + # Storage class; default is local-path + storageClass: + +# Model Settings +models: + # Large language model for Q&A; requires OpenAI-compatible API + answer: + # [Required] API endpoint URL (please check API provider documentation whether to include "v1" suffix) + endpoint: + # [Required] API Key; empty by default + key: + # [Required] Model name + name: + # [Required] Model maximum context length; recommended >=8192 + ctxLength: 8192 + # Model maximum output length, recommended >=2048 + maxTokens: 2048 + # Model for Function Call; recommended to use specific inference framework + functionCall: + # Inference framework type, default is ollama + # Available framework types: ["vllm", "sglang", "ollama", "openai"] + backend: + # [Required] Model endpoint; please check API provider documentation whether to include "v1" suffix + # Leave empty to use same as Q&A model + endpoint: + # API Key; leave empty to use same as Q&A model + key: + # Model name; leave empty to use same as Q&A model + name: + # Model maximum context length; leave empty to use same as Q&A model + ctxLength: + # Model maximum output length; leave empty to use same as Q&A model + maxTokens: + # Model for data embedding + embedding: + # Inference framework type, default is openai + # [Required] Embedding API type: ["openai", "mindie"] + type: + # [Required] Embedding URL (requires "v1" suffix) + endpoint: + # [Required] Embedding model API Key + key: + # [Required] Embedding model name + name: + +# Login Settings +login: + # Client ID settings, only effective when type is authhub + client: + # [Required] Client ID + id: + # [Required] Client secret + secret: + +# Domain Settings +domain: + # [Required] EulerCopilot web frontend URL; default is http://127.0.0.1:30080 + euler_copilot: + # [Required] AuthHub web frontend URL; default is http://127.0.0.1:30081 + authhub: + +# Storage Settings +storage: + # Framework semantics + frameworkSemantics: + # Framework semantics path + path: + # Framework semantics storage size, default is 5GB + size: + # Shared storage + webAsset: + # Frontend/backend shared storage size, default is 10GB + size: + +euler_copilot: + # Configuration file tool + secretInject: + # Image settings; default is hub.oepkgs.net/neocopilot/secret_inject:dev-x86 + # Image tags: ["dev-x86", "dev-arm"] + image: + + framework: + # [Required] Whether to deploy Framework backend service + enabled: true + # Image settings; default is hub.oepkgs.net/neocopilot/euler-copilot-framework:0.10.0-x86 + # Image tags: ["0.10.0-x86", "0.10.0-arm"] + image: + # Container root directory read-only + readOnly: + # Resource limits settings + resourceLimits: {} + # Service settings + service: + # Service type, e.g., NodePort + type: + # When type is nodePort, specify the host port number + nodePort: + + web: + # [Required] Whether to deploy Web frontend user interface + enabled: true + # Image settings; default is hub.oepkgs.net/neocopilot/euler-copilot-web:0.10.0-x86 + # Image tags: ["0.10.0-x86", "0.10.0-arm"] + image: + # Container root directory read-only + readOnly: + # Resource limits settings + resourceLimits: {} + # Service settings + service: + # Service type, e.g., NodePort + type: NodePort + # When type is NodePort, specify the host port number + nodePort: 30080 + + rag_web: + # [Required] Whether to deploy RAG Web frontend user interface + enabled: true + # Image settings; default is hub.oepkgs.net/neocopilot/data_chain_web:0.10.0-x86 + # Image tags: ["0.10.0-x86", "0.10.0-arm"] + image: + # Container root directory read-only + readOnly: + # Resource limits settings + resourceLimits: {} + # Service settings + service: + # Service type, e.g., NodePort + type: + # When type is NodePort, specify the host port number + nodePort: + + rag: + # [Required] Whether to deploy RAG backend service + enabled: true + # Image settings; default is hub.oepkgs.net/neocopilot/data_chain_back_end:0.10.0-x86 + # Image tags: ["0.10.0-x86", "0.10.0-arm"] + image: + # Container root directory read-only + readOnly: + # Resource limits settings + resourceLimits: {} + # Service settings + service: + # Service type, e.g., NodePort + type: + # When type is NodePort, specify the host port number + nodePort: diff --git a/deploy/en/chart/rca/template/rca/rca-anteater/anteater-deployment.yaml b/deploy/en/chart/rca/template/rca/rca-anteater/anteater-deployment.yaml new file mode 100644 index 000000000..23657ebf6 --- /dev/null +++ b/deploy/en/chart/rca/template/rca/rca-anteater/anteater-deployment.yaml @@ -0,0 +1,49 @@ +{{- if .Values.rca.anteater.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: anteater-deploy-{{ .Release.Name }} + namespace: {{ .Release.Namespace }} + labels: + app: anteater-{{ .Release.Name }} +spec: + replicas: 1 + selector: + matchLabels: + app: anteater-{{ .Release.Name }} + template: + metadata: + annotations: + checksum/secret: {{ include (print $.Template.BasePath "/rca/rca-anteater/anteater-secret.yaml") . | sha256sum }} + labels: + app: anteater-{{ .Release.Name }} + spec: + containers: + - name: anteater + image: "{{if ne ( .Values.rca.anteater.image.registry | toString ) ""}}{{ .Values.rca.anteater.image.registry }}{{ else }}{{ .Values.globals.imageRegistry }}{{ end }}/{{ .Values.rca.anteater.image.name }}:{{ .Values.rca.anteater.image.tag | toString }}" + imagePullPolicy: {{ if ne ( .Values.rca.anteater.image.imagePullPolicy | toString ) "" }}{{ .Values.rca.anteater.image.imagePullPolicy }}{{ else }}{{ .Values.globals.imagePullPolicy }}{{ end }} + ports: + - containerPort: 5210 + protocol: TCP + env: + - name: TZ + value: Asia/Shanghai + volumeMounts: + - mountPath: /etc/gala-anteater/config/log.settings.ini + name: anteater-config-volume + subPath: log.settings.ini + - mountPath: /etc/gala-anteater/config/gala-anteater.yaml + name: anteater-config-volume + subPath: gala-anteater.yaml + - mountPath: /home/llm_ops/config/config.json + name: anteater-config-volume + subPath: config.json + - mountPath: /home/gala-anteater/config/module/usad_model.job.json + name: anteater-config-volume + subPath: usad_model.job.json + restartPolicy: Always + volumes: + - name: anteater-config-volume + configMap: + name: anteater-config +{{- end }} \ No newline at end of file diff --git a/deploy/en/chart/rca/template/rca/rca-anteater/anteater-secret.yaml b/deploy/en/chart/rca/template/rca/rca-anteater/anteater-secret.yaml new file mode 100644 index 000000000..33a5c630a --- /dev/null +++ b/deploy/en/chart/rca/template/rca/rca-anteater/anteater-secret.yaml @@ -0,0 +1,14 @@ +{{- if .Values.rca.anteater.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: gala-anteater-{{ .Release.Name }} + namespace: {{ .Release.Namespace }} +spec: + type: ClusterIP + selector: + app: anteater-{{ .Release.Name }} + ports: + - port: 5210 + targetPort: 5210 +{{- end }} \ No newline at end of file diff --git a/deploy/en/chart_ssl/traefik-config.yml b/deploy/en/chart_ssl/traefik-config.yml new file mode 100644 index 000000000..3af36e4aa --- /dev/null +++ b/deploy/en/chart_ssl/traefik-config.yml @@ -0,0 +1,15 @@ +apiVersion: helm.cattle.io/v1 +kind: HelmChartConfig +metadata: + name: traefik + namespace: kube-system +spec: + valuesContent: |- + ports: + web: + expose: false + exposedPort: 8083 + websecure: + expose: true + exposedPort: 443 + diff --git a/deploy/en/chart_ssl/traefik-secret.yaml b/deploy/en/chart_ssl/traefik-secret.yaml new file mode 100644 index 000000000..04126d326 --- /dev/null +++ b/deploy/en/chart_ssl/traefik-secret.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Secret +metadata: + name: traefik-ssl-secret + namespace: default #Must be default +stringData: + tls.crt: | + # Paste certificate public key chain here + + tls.key: | + # Paste certificate private key here diff --git a/deploy/en/chart_ssl/traefik-tlsstore.yaml b/deploy/en/chart_ssl/traefik-tlsstore.yaml new file mode 100644 index 000000000..7e948f8e4 --- /dev/null +++ b/deploy/en/chart_ssl/traefik-tlsstore.yaml @@ -0,0 +1,9 @@ +apiVersion: traefik.containo.us/v1alpha1 +kind: TLSStore +metadata: + name: default + namespace: default + +spec: + defaultCertificate: + secretName: traefik-ssl-secret diff --git a/deploy/en/scripts/0-one-click-deploy/one-click-deploy.sh b/deploy/en/scripts/0-one-click-deploy/one-click-deploy.sh new file mode 100755 index 000000000..f44178e41 --- /dev/null +++ b/deploy/en/scripts/0-one-click-deploy/one-click-deploy.sh @@ -0,0 +1,294 @@ +#!/bin/bash + +# Enhanced color definitions +RESET='\033[0m' +BOLD='\033[1m' +RED='\033[38;5;196m' +GREEN='\033[38;5;46m' +YELLOW='\033[38;5;226m' +BLUE='\033[38;5;45m' +MAGENTA='\033[38;5;201m' +CYAN='\033[38;5;51m' +WHITE='\033[38;5;255m' +BG_RED='\033[48;5;196m' +BG_GREEN='\033[48;5;46m' +BG_BLUE='\033[48;5;45m' +DIM='\033[2m' + +# Progress bar width +PROGRESS_WIDTH=50 +NAMESPACE="euler-copilot" +TIMEOUT=300 # Max wait time (seconds) +INTERVAL=10 # Check interval (seconds) + +# Global variables +authhub_address="" +eulercopilot_address="" + +# Parse command line arguments +parse_arguments() { + while [[ $# -gt 0 ]]; do + case "$1" in + --eulercopilot_address) + if [ -n "$2" ]; then + eulercopilot_address="$2" + shift 2 + else + echo -e "${RED}Error: --eulercopilot_address requires a value${RESET}" >&2 + exit 1 + fi + ;; + --authhub_address) + if [ -n "$2" ]; then + authhub_address="$2" + shift 2 + else + echo -e "${RED}Error: --authhub_address requires a value${RESET}" >&2 + exit 1 + fi + ;; + *) + echo -e "${RED}Unknown option: $1${RESET}" >&2 + exit 1 + ;; + esac + done +} + +# Prompt user for required parameters +prompt_for_addresses() { + # If eulercopilot_address not provided via command line, prompt user + if [ -z "$eulercopilot_address" ]; then + echo -e "${YELLOW}EulerCopilot address not provided${RESET}" + read -p "$(echo -e "${CYAN}Enter EulerCopilot address (e.g., http://myhost:30080): ${RESET}")" eulercopilot_address + + # Validate input not empty + while [ -z "$eulercopilot_address" ]; do + echo -e "${RED}Error: EulerCopilot address cannot be empty${RESET}" + read -p "$(echo -e "${CYAN}Enter EulerCopilot address (e.g., http://myhost:30080): ${RESET}")" eulercopilot_address + done + fi + + # If authhub_address not provided via command line, prompt user + if [ -z "$authhub_address" ]; then + echo -e "${YELLOW}Authhub address not provided${RESET}" + read -p "$(echo -e "${CYAN}Enter Authhub address (e.g., http://myhost:30081): ${RESET}")" authhub_address + + # Validate input not empty + while [ -z "$authhub_address" ]; do + echo -e "${RED}Error: Authhub address cannot be empty${RESET}" + read -p "$(echo -e "${CYAN}Enter Authhub address (e.g., http://myhost:30081): ${RESET}")" authhub_address + done + fi +} + +# Colorful progress bar function +colorful_progress() { + local current=$1 + local total=$2 + local progress=$((current*100/total)) + local completed=$((PROGRESS_WIDTH*current/total)) + local remaining=$((PROGRESS_WIDTH-completed)) + + printf "\r${BOLD}${BLUE}⟦${RESET}" + printf "${BG_BLUE}${WHITE}%${completed}s${RESET}" | tr ' ' '▌' + printf "${DIM}${BLUE}%${remaining}s${RESET}" | tr ' ' '·' + printf "${BOLD}${BLUE}⟧${RESET} ${GREEN}%3d%%${RESET} ${CYAN}[%d/%d]${RESET}" \ + $progress $current $total +} + +# Print separator line +print_separator() { + echo -e "${BLUE}${BOLD}$(printf '━%.0s' $(seq 1 $(tput cols)))${RESET}" +} + +# Print step title +print_step_title() { + echo -e "\n${BG_BLUE}${WHITE}${BOLD} Step $1 ${RESET} ${MAGENTA}${BOLD}$2${RESET}" + echo -e "${DIM}${BLUE}$(printf '━%.0s' $(seq 1 $(tput cols)))${RESET}" +} + +# Get main script absolute path and switch to its directory +MAIN_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +cd "$MAIN_DIR" || exit 1 + +run_script_with_check() { + local script_path=$1 + local script_name=$2 + local step_number=$3 + local auto_input=${4:-false} + shift 4 + local extra_args=("$@") # Use array for extra parameters + + # Pre-check: script exists + if [ ! -f "$script_path" ]; then + echo -e "\n${BOLD}${RED}✗ Fatal error:${RESET}${YELLOW}${script_name}${RESET}${RED} not found (path: ${CYAN}${script_path}${RED})${RESET}" >&2 + exit 1 + fi + + print_step_title $step_number "$script_name" + + # Get absolute path and execution directory + local script_abs_path=$(realpath "$script_path") + local script_dir=$(dirname "$script_abs_path") + local script_base=$(basename "$script_abs_path") + + echo -e "${DIM}${BLUE}🠖 Script path: ${YELLOW}${script_abs_path}${RESET}" + echo -e "${DIM}${BLUE}🠖 Working directory: ${YELLOW}${script_dir}${RESET}" + echo -e "${DIM}${BLUE}🠖 Extra args: ${YELLOW}${extra_args[*]}${RESET}" + echo -e "${DIM}${BLUE}🠖 Start time: ${YELLOW}$(date +'%Y-%m-%d %H:%M:%S')${RESET}" + + # Create temporary log file + local log_file=$(mktemp) + echo -e "${DIM}${BLUE}🠖 Temp log: ${YELLOW}${log_file}${RESET}" + + # Execute script (with auto-input and real-time logging) + local exit_code=0 + if $auto_input; then + (cd "$script_dir" && yes "" | bash "./$script_base" "${extra_args[@]}" 2>&1 | tee "$log_file") + else + (cd "$script_dir" && bash "./$script_base" "${extra_args[@]}" 2>&1 | tee "$log_file") + fi + exit_code=${PIPESTATUS[0]} + + # Handle execution result + if [ $exit_code -eq 0 ]; then + echo -e "\n${BOLD}${GREEN}✓ ${script_name} executed successfully!${RESET}" + echo -e "${DIM}${CYAN}$(printf '%.0s─' $(seq 1 $(tput cols)))${RESET}" + echo -e "${DIM}${CYAN}Operation log:${RESET}" + cat "$log_file" | sed -e "s/^/${DIM}${CYAN} 🠖 ${RESET}/" + echo -e "${DIM}${CYAN}$(printf '%.0s─' $(seq 1 $(tput cols)))${RESET}" + else + echo -e "\n${BOLD}${RED}✗ ${script_name} execution failed!${RESET}" >&2 + echo -e "${DIM}${RED}$(printf '%.0s─' $(seq 1 $(tput cols)))${RESET}" >&2 + echo -e "${DIM}${RED}Error log:${RESET}" >&2 + cat "$log_file" | sed -e "s/^/${DIM}${RED} ✗ ${RESET}/" >&2 + echo -e "${DIM}${RED}$(printf '%.0s─' $(seq 1 $(tput cols)))${RESET}" >&2 + rm "$log_file" + exit 1 + fi + + rm "$log_file" + return $exit_code +} + +# Uninstall all components +uninstall_all() { + echo -e "\n${CYAN}▸ Uninstalling all Helm Releases...${RESET}" + local RELEASES + RELEASES=$(helm list -n $NAMESPACE --short 2>/dev/null || true) + + if [ -n "$RELEASES" ]; then + echo -e "${YELLOW}Found Helm Releases:${RESET}" + echo "$RELEASES" | awk '{print " ➤ "$0}' + for release in $RELEASES; do + echo -e "${BLUE}Deleting: ${release}${RESET}" + helm uninstall "$release" -n $NAMESPACE || echo -e "${RED}Delete failed, continuing...${RESET}" + done + else + echo -e "${YELLOW}No Helm Releases to clean${RESET}" + fi + + echo -e "\n${CYAN}▸ Cleaning persistent storage...${RESET}" + local pvc_list + pvc_list=$(kubectl get pvc -n $NAMESPACE -o name 2>/dev/null || true) + + if [ -n "$pvc_list" ]; then + echo -e "${YELLOW}Found PVC resources:${RESET}" + echo "$pvc_list" | awk '{print " ➤ "$0}' + echo "$pvc_list" | xargs -n 1 kubectl delete -n $NAMESPACE || echo -e "${RED}Delete failed, continuing...${RESET}" + else + echo -e "${YELLOW}No PVCs to clean${RESET}" + fi + + echo -e "\n${CYAN}▸ Cleaning Secret resources...${RESET}" + local secret_list + secret_list=$(kubectl get secret -n $NAMESPACE -o name 2>/dev/null || true) + + if [ -n "$secret_list" ]; then + echo -e "${YELLOW}Found Secret resources:${RESET}" + echo "$secret_list" | awk '{print " ➤ "$0}' + echo "$secret_list" | xargs -n 1 kubectl delete -n $NAMESPACE || echo -e "${RED}Delete failed, continuing...${RESET}" + else + echo -e "${YELLOW}No Secrets to clean${RESET}" + fi + + echo -e "\n${BG_GREEN}${WHITE}${BOLD} ✓ Complete ${RESET} ${GREEN}All resources cleaned${RESET}" +} + +# Main header display +show_header() { + clear + echo -e "\n${BOLD}${MAGENTA}$(printf '✧%.0s' $(seq 1 $(tput cols)))${RESET}" + echo -e "${BOLD}${WHITE} Euler Copilot One-Click Deployment System ${RESET}" + echo -e "${BOLD}${MAGENTA}$(printf '✧%.0s' $(seq 1 $(tput cols)))${RESET}" + echo -e "${CYAN}◈ Main directory: ${YELLOW}${MAIN_DIR}${RESET}" + echo -e "${CYAN}◈ EulerCopilot address: ${YELLOW}${eulercopilot_address:-Not set}${RESET}" + echo -e "${CYAN}◈ Authhub address: ${YELLOW}${authhub_address:-Not set}${RESET}\n" +} + +# Modified start_deployment function step configuration +start_deployment() { + local total_steps=8 + local current_step=1 + + # Step configuration (script_path script_name auto_input extra_args_array) + local steps=( + "../1-check-env/check_env.sh Environment check false" + "_conditional_tools_step Basic tools installation(k3s+helm) true" + "../3-install-ollama/install_ollama.sh Ollama deployment true" + "../4-deploy-deepseek/deploy_deepseek.sh Deepseek model deployment false" + "../5-deploy-embedding/deploy-embedding.sh Embedding service deployment false" + "../6-install-databases/install_databases.sh Database cluster deployment false" + "../7-install-authhub/install_authhub.sh Authhub deployment true --authhub_address ${authhub_address}" + "_conditional_eulercopilot_step EulerCopilot deployment true" + ) + + for step in "${steps[@]}"; do + local script_path=$(echo "$step" | awk '{print $1}') + local script_name=$(echo "$step" | awk '{print $2}') + local auto_input=$(echo "$step" | awk '{print $3}') + local extra_args=$(echo "$step" | awk '{for(i=4;i<=NF;i++) printf $i" "}') + + # Special step handling + if [[ "$script_path" == "_conditional_tools_step" ]]; then + handle_tools_step $current_step + elif [[ "$script_path" == "_conditional_eulercopilot_step" ]]; then + sleep 60 + handle_eulercopilot_step $current_step + else + run_script_with_check "$script_path" "$script_name" $current_step $auto_input $extra_args + fi + + colorful_progress $current_step $total_steps + ((current_step++)) + done +} + +# Handle tools installation step +handle_tools_step() { + local current_step=$1 + if command -v k3s >/dev/null 2>&1 && command -v helm >/dev/null 2>&1; then + echo -e "${CYAN}🠖 k3s and helm detected, performing environment cleanup...${RESET}" + uninstall_all + else + run_script_with_check "../2-install-tools/install_tools.sh" "Basic tools installation" $current_step true + fi +} + +handle_eulercopilot_step() { + local current_step=$1 + local extra_args=() + + # Build extra arguments array + [ -n "$authhub_address" ] && extra_args+=(--authhub_address "$authhub_address") + [ -n "$eulercopilot_address" ] && extra_args+=(--eulercopilot_address "$eulercopilot_address") + + run_script_with_check "../8-install-EulerCopilot/install_eulercopilot.sh" "EulerCopilot deployment" $current_step true "${extra_args[@]}" +} + +# Main execution flow +parse_arguments "$@" +prompt_for_addresses +show_header +start_deployment diff --git a/deploy/en/scripts/1-check-env/check_env.sh b/deploy/en/scripts/1-check-env/check_env.sh new file mode 100755 index 000000000..0ab87a24c --- /dev/null +++ b/deploy/en/scripts/1-check-env/check_env.sh @@ -0,0 +1,250 @@ +#!/bin/bash + +# Color definitions +COLOR_INFO='\033[34m' # Blue info +COLOR_SUCCESS='\033[32m' # Green success +COLOR_ERROR='\033[31m' # Red error +COLOR_WARNING='\033[33m' # Yellow warning +COLOR_RESET='\033[0m' # Reset color + +# Global mode flag +OFFLINE_MODE=false + +function check_user { + if [[ $(id -u) -ne 0 ]]; then + echo -e "${COLOR_ERROR}[Error] Please run this script with root privileges!${COLOR_RESET}" + return 1 + fi + return 0 +} + +function check_version { + local current_version_id="$1" + local supported_versions=("${@:2}") + + echo -e "${COLOR_INFO}[Info] Current OS version: $current_version_id${COLOR_RESET}" + for version_id in "${supported_versions[@]}"; do + if [[ "$current_version_id" == "$version_id" ]]; then + echo -e "${COLOR_SUCCESS}[Success] OS meets compatibility requirements${COLOR_RESET}" + return 0 + fi + done + + echo -e "${COLOR_ERROR}[Error] OS does not meet compatibility requirements, script will exit${COLOR_RESET}" + return 1 +} + +function check_os_version { + local id=$(grep '^ID=' /etc/os-release | cut -d= -f2 | tr -d '"') + local version=$(grep -E "^VERSION_ID=" /etc/os-release | cut -d '"' -f 2) + + echo -e "${COLOR_INFO}[Info] Current distribution: $id${COLOR_RESET}" + + case $id in + "openEuler"|"bclinux") + local supported_versions=("22.03" "22.09" "23.03" "23.09" "24.03") + check_version "$version" "${supported_versions[@]}" + ;; + "InLinux") + local supported_versions=("23.12") + check_version "$version" "${supported_versions[@]}" + ;; + "FusionOS") + local supported_versions=("23") + check_version "$version" "${supported_versions[@]}" + ;; + "uos") + local supported_versions=("20") + check_version "$version" "${supported_versions[@]}" + ;; + "HopeOS") + local supported_versions=("V22") + check_version "$version" "${supported_versions[@]}" + ;; + "kylin") + local supported_versions=("V10") + check_version "$version" "${supported_versions[@]}" + ;; + *) + echo -e "${COLOR_ERROR}[Error] Distribution not supported, script will exit${COLOR_RESET}" + return 1 + ;; + esac + return $? +} + +function check_hostname { + local current_hostname=$(cat /etc/hostname) + if [[ -z "$current_hostname" ]]; then + echo -e "${COLOR_ERROR}[Error] Hostname not set, automatically set to localhost${COLOR_RESET}" + set_hostname "localhost" + return $? + else + echo -e "${COLOR_INFO}[Info] Current hostname: $current_hostname${COLOR_RESET}" + echo -e "${COLOR_SUCCESS}[Success] Hostname is set${COLOR_RESET}" + return 0 + fi +} + +function set_hostname { + if ! command -v hostnamectl &> /dev/null; then + echo "$1" > /etc/hostname + echo -e "${COLOR_SUCCESS}[Success] Manual hostname set successful${COLOR_RESET}" + return 0 + fi + + if hostnamectl set-hostname "$1"; then + echo -e "${COLOR_SUCCESS}[Success] Hostname set successfully${COLOR_RESET}" + return 0 + else + echo -e "${COLOR_ERROR}[Error] Hostname set failed${COLOR_RESET}" + return 1 + fi +} + +function check_dns { + echo -e "${COLOR_INFO}[Info] Checking DNS settings${COLOR_RESET}" + if grep -q "^nameserver" /etc/resolv.conf; then + echo -e "${COLOR_SUCCESS}[Success] DNS configured${COLOR_RESET}" + return 0 + fi + + if $OFFLINE_MODE; then + echo -e "${COLOR_WARNING}[Warning] Offline mode: Please manually configure internal DNS server${COLOR_RESET}" + return 0 + else + echo -e "${COLOR_ERROR}[Error] DNS not configured, automatically set to 8.8.8.8${COLOR_RESET}" + set_dns "8.8.8.8" + return $? + fi +} + +function set_dns { + if systemctl is-active --quiet NetworkManager; then + local net_ic=$(nmcli -t -f NAME con show --active | head -n 1) + if [[ -z "$net_ic" ]]; then + echo -e "${COLOR_ERROR}[Error] No active network connection found${COLOR_RESET}" + return 1 + fi + + if nmcli con mod "$net_ic" ipv4.dns "$1" && nmcli con mod "$net_ic" ipv4.ignore-auto-dns yes; then + nmcli con down "$net_ic" && nmcli con up "$net_ic" + echo -e "${COLOR_SUCCESS}[Success] DNS set successfully${COLOR_RESET}" + return 0 + else + echo -e "${COLOR_ERROR}[Error] DNS set failed${COLOR_RESET}" + return 1 + fi + else + cp /etc/resolv.conf /etc/resolv.conf.bak + echo "nameserver $1" >> /etc/resolv.conf + echo -e "${COLOR_SUCCESS}[Success] Manual DNS set successful${COLOR_RESET}" + return 0 + fi +} + +function check_ram { + local RAM_THRESHOLD=16000 + local current_mem=$(free -m | awk '/Mem/{print $2}') + + echo -e "${COLOR_INFO}[Info] Current memory: $current_mem MB${COLOR_RESET}" + if (( current_mem < RAM_THRESHOLD )); then + echo -e "${COLOR_ERROR}[Error] Insufficient memory ${RAM_THRESHOLD} MB${COLOR_RESET}" + return 1 + fi + echo -e "${COLOR_SUCCESS}[Success] Memory meets requirements${COLOR_RESET}" + return 0 +} + +check_disk_space() { + local DIR="$1" + local THRESHOLD="$2" + + local USAGE=$(df --output=pcent "$DIR" | tail -n 1 | sed 's/%//g' | tr -d ' ') + + if [ "$USAGE" -ge "$THRESHOLD" ]; then + echo -e "${COLOR_WARNING}[Warning] Disk usage for $DIR has reached ${USAGE}%, exceeding threshold ${THRESHOLD}%${COLOR_RESET}" + return 1 + else + echo -e "${COLOR_INFO}[Info] Disk usage for $DIR is ${USAGE}%, below threshold ${THRESHOLD}%${COLOR_RESET}" + return 0 + fi +} + +function check_network { + echo -e "${COLOR_INFO}[Info] Checking network connection...${COLOR_RESET}" + + # Use TCP check instead of curl + if timeout 5 bash -c 'cat < /dev/null > /dev/tcp/www.baidu.com/80' 2>/dev/null; then + echo -e "${COLOR_SUCCESS}[Success] Network connection normal${COLOR_RESET}" + return 0 + else + echo -e "${COLOR_ERROR}[Error] Cannot access external network${COLOR_RESET}" + return 1 + fi +} + +function check_selinux { + sed -i 's/^SELINUX=.*/SELINUX=disabled/g' /etc/selinux/config + echo -e "${COLOR_SUCCESS}[Success] SELinux configuration disabled${COLOR_RESET}" + setenforce 0 &>/dev/null + echo -e "${COLOR_SUCCESS}[Success] SELinux temporarily disabled${COLOR_RESET}" + return 0 +} + +function check_firewall { + systemctl disable --now firewalld &>/dev/null + echo -e "${COLOR_SUCCESS}[Success] Firewall closed and disabled${COLOR_RESET}" + return 0 +} + +function prepare_offline { + echo -e "${COLOR_INFO}[Info] Preparing offline deployment environment..." + mkdir -p /home/eulercopilot/images + mkdir -p /home/eulercopilot/tools + mkdir -p /home/eulercopilot/models + echo -e "1. Please ensure offline installation images are uploaded to /home/eulercopilot/images" + echo -e "2. Please confirm local software repository is configured" + echo -e "3. All tool packages pre-downloaded to local directory /home/eulercopilot/tools" + echo -e "4. All model files pre-downloaded to local directory /home/eulercopilot/models${COLOR_RESET}" +} + +function main { + check_user || return 1 + check_os_version || return 1 + check_hostname || return 1 + + # Network check and mode determination + if check_network; then + OFFLINE_MODE=false + else + OFFLINE_MODE=true + echo -e "${COLOR_WARNING}[Warning] Switching to offline deployment mode${COLOR_RESET}" + prepare_offline + fi + + check_dns || return 1 + check_ram || return 1 + check_disk_space "/" 70 + + if [ $? -eq 1 ]; then + echo -e "${COLOR_WARNING}[Warning] Disk space cleanup required!${COLOR_RESET}" + else + echo -e "${COLOR_SUCCESS}[Success] Disk space normal${COLOR_RESET}" + fi + + check_selinux || return 1 + check_firewall || return 1 + + # Final deployment prompt + echo -e "\n${COLOR_SUCCESS}#####################################" + if $OFFLINE_MODE; then + echo -e "# Environment check complete, ready for offline deployment #" + else + echo -e "# Environment check complete, ready for online deployment #" + fi + echo -e "#####################################${COLOR_RESET}" + return 0 +} + +main diff --git a/deploy/en/scripts/2-install-tools/install_tools.sh b/deploy/en/scripts/2-install-tools/install_tools.sh new file mode 100755 index 000000000..4346d0b82 --- /dev/null +++ b/deploy/en/scripts/2-install-tools/install_tools.sh @@ -0,0 +1,381 @@ +#!/bin/bash + +GITHUB_MIRROR="https://gh-proxy.com" +ARCH=$(uname -m) +TOOLS_DIR="/home/eulercopilot/tools" +eulercopilot_version=0.10.0 + +SCRIPT_PATH="$( + cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 + pwd +)/$(basename "${BASH_SOURCE[0]}")" + +IMPORT_SCRIPT="$( + canonical_path=$(readlink -f "$SCRIPT_PATH" 2>/dev/null || echo "$SCRIPT_PATH") + dirname "$(dirname "$canonical_path")" +)" + +# Function: Display help information +function help { + echo -e "Usage: bash install_tools.sh [options]" + echo -e "Options:" + echo -e " --mirror cn Use China mirror for acceleration" + echo -e " --k3s-version VERSION Specify k3s version (default: v1.30.2+k3s1)" + echo -e " --helm-version VERSION Specify helm version (default: v3.15.0)" + echo -e " -h, --help Show help information" + echo -e "Examples:" + echo -e " bash install_tools.sh # Install with default settings" + echo -e " bash install_tools.sh --mirror cn # Use China mirror" + echo -e " bash install_tools.sh --k3s-version v1.30.1+k3s1 --helm-version v3.15.0" + echo -e "Offline installation instructions:" + echo -e "1. Rename k3s binary to k3s or k3s-arm64 and place in $TOOLS_DIR" + echo -e "2. Rename k3s image package to k3s-airgap-images-.tar.zst and place in $TOOLS_DIR" + echo -e "3. Rename helm package to helm--linux-.tar.gz and place in $TOOLS_DIR" +} + +# Parse command line arguments +MIRROR="" +K3S_VERSION="" +HELM_VERSION="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --mirror) + MIRROR="$2" + shift 2 + ;; + --k3s-version) + K3S_VERSION="$2" + shift 2 + ;; + --helm-version) + HELM_VERSION="$2" + shift 2 + ;; + cn) + MIRROR="cn" + shift + ;; + -h|--help) + help + exit 0 + ;; + *) + echo "Unknown parameter: $1" + help + exit 1 + ;; + esac +done + +# Enhanced network check +function check_network { + echo -e "[Info] Checking network connection..." + if curl --retry 3 --retry-delay 2 --connect-timeout 5 -Is https://github.com | head -n 1 | grep 200 >/dev/null; then + echo -e "\033[32m[OK] Network connection normal\033[0m" + return 0 + else + echo -e "\033[33m[Warn] No network connection, switching to offline mode\033[0m" + return 1 + fi +} + +function check_user { + if [[ $(id -u) -ne 0 ]]; then + echo -e "\033[31m[Error] Please run this script with root privileges!\033[0m" + exit 1 + fi +} + +function check_arch { + case $ARCH in + x86_64) ARCH=amd64 ;; + aarch64) ARCH=arm64 ;; + *) + echo -e "\033[31m[Error] Current CPU architecture not supported\033[0m" + return 1 + ;; + esac + return 0 +} + +install_basic_tools() { + # Install basic tools + echo "Installing tar, vim, curl, wget..." + yum install -y tar vim curl wget python3 + + # Check if pip is installed + if ! command -v pip3 &> /dev/null; then + echo -e "pip could not be found, installing python3-pip..." + yum install -y python3-pip + else + echo -e "pip is already installed." + fi + + echo "Installing requests ruamel.yaml with pip3..." + if ! pip3 install \ + --disable-pip-version-check \ + --retries 3 \ + --timeout 60 \ + --trusted-host mirrors.huaweicloud.com \ + -i https://mirrors.huaweicloud.com/repository/pypi/simple \ + ruamel.yaml requests; then + echo -e "[ERROR] Failed to install ruamel.yaml and requests via pip" >&2 + fi + echo "All basic tools have been installed." + return 0 +} + +function check_local_k3s_files { + local version="${1:-v1.30.2+k3s1}" + local k3s_version="$version" + + # Auto-complete v prefix + if [[ $k3s_version != v* ]]; then + k3s_version="v$k3s_version" + fi + + local image_name="k3s-airgap-images-$ARCH.tar.zst" + local bin_name="k3s" + [[ $ARCH == "arm64" ]] && bin_name="k3s-arm64" + + # Check if local files exist + if [[ -f "$TOOLS_DIR/$bin_name" && -f "$TOOLS_DIR/$image_name" ]]; then + echo -e "\033[32m[Info] Local K3s installation files detected, using local files\033[0m" + return 0 + else + echo -e "\033[33m[Info] Local K3s installation files incomplete, attempting online download\033[0m" + return 1 + fi +} + +function install_k3s { + local version="${1:-v1.30.2+k3s1}" + local use_mirror="$2" + + # Auto-complete v prefix + if [[ $version != v* ]]; then + version="v$version" + fi + local k3s_version="$version" + + local image_name="k3s-airgap-images-$ARCH.tar.zst" + local bin_name="k3s" + [[ $ARCH == "arm64" ]] && bin_name="k3s-arm64" + + # First check if local files exist + if check_local_k3s_files "$version"; then + # Use local files for installation + echo -e "\033[33m[Info] Entering offline K3s installation mode\033[0m" + + echo -e "[Info] Installing using local packages..." + cp "$TOOLS_DIR/$bin_name" /usr/local/bin/k3s + chmod +x /usr/local/bin/k3s + + mkdir -p /var/lib/rancher/k3s/agent/images + cp "$TOOLS_DIR/$image_name" "/var/lib/rancher/k3s/agent/images/$image_name" + + # Offline installation script + local local_install_script="$TOOLS_DIR/k3s-install.sh" + if [[ -f "$local_install_script" ]]; then + echo -e "\033[33m[Info] Using local installation script: $local_install_script\033[0m" + chmod +x "$local_install_script" + if INSTALL_K3S_SKIP_DOWNLOAD=true "$local_install_script"; then + echo -e "\033[32m[Success] K3s installation completed\033[0m" + return 0 + else + echo -e "\033[31m[Error] Local installation failed\033[0m" + return 1 + fi + else + echo -e "\033[31m[Error] Missing local installation script: $local_install_script\033[0m" + echo -e "Please pre-download and save to specified directory:" + echo -e "Online: curl -sfL https://get.k3s.io -o $local_install_script" + echo -e "China mirror: curl -sfL https://rancher-mirror.rancher.cn/k3s/k3s-install.sh -o $local_install_script" + return 1 + fi + else + # Local files not found, check network + if check_network; then + echo -e "\033[32m[Info] Starting online K3s installation\033[0m" + + # Online download and installation + local k3s_bin_url="$GITHUB_MIRROR/https://github.com/k3s-io/k3s/releases/download/$k3s_version/$bin_name" + local k3s_image_url="$GITHUB_MIRROR/https://github.com/k3s-io/k3s/releases/download/$k3s_version/$image_name" + + echo -e "[Info] Downloading K3s binary..." + if ! curl -L "$k3s_bin_url" -o /usr/local/bin/k3s; then + echo -e "\033[31m[Error] Binary download failed\033[0m" + return 1 + fi + chmod +x /usr/local/bin/k3s + + echo -e "[Info] Downloading dependency images..." + mkdir -p /var/lib/rancher/k3s/agent/images + if ! curl -L "$k3s_image_url" -o "/var/lib/rancher/k3s/agent/images/$image_name"; then + echo -e "\033[33m[Warn] Image download failed, may affect offline capability\033[0m" + fi + + local install_source="https://get.k3s.io" + [[ $use_mirror == "cn" ]] && install_source="https://rancher-mirror.rancher.cn/k3s/k3s-install.sh" + + echo -e "\033[32m[Info] Using online installation script\033[0m" + if ! curl -sfL "$install_source" | INSTALL_K3S_SKIP_DOWNLOAD=true sh -; then + echo -e "\033[31m[Error] Online installation failed\033[0m" + return 1 + fi + else + # Neither local files nor network connection available + echo -e "\033[31m[Error] Cannot install K3s:\033[0m" + echo -e "1. Missing necessary local installation files" + echo -e "2. Network unavailable, cannot download installation files" + echo -e "Please perform one of the following:" + echo -e "- Ensure network connection and retry" + echo -e "- Or pre-place the following files in $TOOLS_DIR directory:" + echo -e " - $bin_name" + echo -e " - $image_name" + echo -e " - k3s-install.sh (optional)" + return 1 + fi + fi +} + +function check_local_helm_file { + local version="${1:-v3.15.0}" + local helm_version="$version" + + # Auto-complete v prefix + if [[ $helm_version != v* ]]; then + helm_version="v$helm_version" + fi + + local file_name="helm-${helm_version}-linux-${ARCH}.tar.gz" + + # Check if local file exists + if [[ -f "$TOOLS_DIR/$file_name" ]]; then + echo -e "\033[32m[Info] Local Helm installation file detected, using local file\033[0m" + return 0 + else + echo -e "\033[33m[Info] Local Helm installation file not found, attempting online download\033[0m" + return 1 + fi +} + +function install_helm { + local version="${1:-v3.15.0}" + local use_mirror="$2" + + # Auto-complete v prefix + if [[ $version != v* ]]; then + version="v$version" + fi + local helm_version="$version" + + local file_name="helm-${helm_version}-linux-${ARCH}.tar.gz" + + # First check if local file exists + if check_local_helm_file "$version"; then + echo -e "\033[33m[Info] Entering offline Helm installation mode\033[0m" + echo -e "[Info] Installing using local package..." + cp "$TOOLS_DIR/$file_name" . + else + # Local file not found, check network + if check_network; then + echo -e "\033[32m[Info] Starting online Helm installation\033[0m" + + local base_url="https://get.helm.sh" + if [[ $use_mirror == "cn" ]]; then + local helm_version_without_v="${helm_version#v}" + base_url="https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts/helm/${helm_version_without_v}" + fi + + echo -e "[Info] Downloading Helm..." + if ! curl -L "$base_url/$file_name" -o "$file_name"; then + echo -e "\033[31m[Error] Download failed\033[0m" + return 1 + fi + else + # Neither local file nor network connection available + echo -e "\033[31m[Error] Cannot install Helm:\033[0m" + echo -e "1. Missing necessary local installation file" + echo -e "2. Network unavailable, cannot download installation file" + echo -e "Please perform one of the following:" + echo -e "- Ensure network connection and retry" + echo -e "- Or pre-place the following file in $TOOLS_DIR directory:" + echo -e " - $file_name" + return 1 + fi + fi + + echo -e "[Info] Extracting and installing..." + tar -zxvf "$file_name" --strip-components 1 -C /usr/local/bin "linux-$ARCH/helm" + chmod +x /usr/local/bin/helm + rm -f "$file_name" + + echo -e "\033[32m[Success] Helm installation completed\033[0m" + return 0 +} + +function check_k3s_status() { + local STATUS=$(systemctl is-active k3s) + + if [ "$STATUS" = "active" ]; then + echo -e "[Info] k3s service is currently active." + else + echo -e "[Info] k3s service is not active, current status: $STATUS. Attempting to start service..." + # Try to start k3s service + systemctl start k3s.service + + # Check service status again + STATUS=$(systemctl is-active k3s.service) + if [ "$STATUS" = "active" ]; then + echo -e "[Info] k3s service successfully started and running." + else + echo -e "\033[31m[Error] Unable to start k3s service, please check logs or configuration\033[0m" + fi + fi +} + +function main { + # Create tools directory + mkdir -p "$TOOLS_DIR" + + check_user + check_arch || exit 1 + install_basic_tools + + local use_mirror="$MIRROR" + local k3s_version="${K3S_VERSION:-v1.30.2+k3s1}" + local helm_version="${HELM_VERSION:-v3.15.0}" + + # Install K3s (if not already installed) + if ! command -v k3s &> /dev/null; then + install_k3s "$k3s_version" "$use_mirror" || exit 1 + else + echo -e "[Info] K3s already installed, skipping installation" + fi + # Check network first + if check_network; then + echo -e "\033[32m[Info] Online environment, skipping image import\033[0m" + else + echo -e "\033[33m[Info] Offline environment, starting local image import, ensure all image files exist in local directory\033[0m" + bash "$IMPORT_SCRIPT/9-other-script/import_images.sh" -v "$eulercopilot_version" + fi + + # Install Helm (if not already installed) + if ! command -v helm &> /dev/null; then + install_helm "$helm_version" "$use_mirror" || exit 1 + else + echo -e "[Info] Helm already installed, skipping installation" + fi + mkdir -p ~/.kube + ln -sf /etc/rancher/k3s/k3s.yaml ~/.kube/config + check_k3s_status + + echo -e "\n\033[32m=== All tools installation completed ===\033[0m" + echo -e "K3s version: $(k3s --version | head -n1)" + echo -e "Helm version: $(helm version --short)" +} + +# Execute main function +main diff --git a/deploy/en/scripts/3-install-ollama/install_ollama.sh b/deploy/en/scripts/3-install-ollama/install_ollama.sh new file mode 100755 index 000000000..8aedaf8d2 --- /dev/null +++ b/deploy/en/scripts/3-install-ollama/install_ollama.sh @@ -0,0 +1,337 @@ +#!/bin/bash + +MAGENTA='\e[35m' +CYAN='\e[36m' +BLUE='\e[34m' +GREEN='\e[32m' +YELLOW='\e[33m' +RED='\e[31m' +RESET='\e[0m' + +# Initialize global variables +OS_ID="" +ARCH="" +OLLAMA_BIN_PATH="/usr/bin/ollama" +OLLAMA_LIB_DIR="/usr/lib/ollama" +OLLAMA_DATA_DIR="/var/lib/ollama" +SERVICE_FILE="/etc/systemd/system/ollama.service" +LOCAL_DIR="/home/eulercopilot/tools" +LOCAL_TGZ="ollama-linux-${ARCH}.tgz" + +# Output function with timestamp +log() { + local level=$1 + shift + local color + case "$level" in + "INFO") color=${BLUE} ;; + "SUCCESS") color=${GREEN} ;; + "WARNING") color=${YELLOW} ;; + "ERROR") color=${RED} ;; + *) color=${RESET} ;; + esac + echo -e "${color}[$(date '+%Y-%m-%d %H:%M:%S')] $level: $*${RESET}" +} + +# Network connection check +check_network() { + local install_url=$(get_ollama_url) + local domain=$(echo "$install_url" | awk -F/ '{print $3}') + local test_url="http://$domain" + + log "INFO" "Checking network connection ($domain)..." + if curl --silent --head --fail --connect-timeout 5 --max-time 10 "$test_url" >/dev/null 2>&1; then + log "INFO" "Network connection normal" + return 0 + else + log "WARNING" "Unable to connect to internet" + return 1 + fi +} + +# Operating system detection +detect_os() { + log "INFO" "Step 1/8: Detecting operating system and architecture..." + if [ -f /etc/os-release ]; then + . /etc/os-release + OS_ID="${ID}" + log "INFO" "Detected operating system: ${PRETTY_NAME}" + else + log "ERROR" "Unable to detect operating system type" + exit 1 + fi + + ARCH=$(uname -m) + case "$ARCH" in + x86_64) ARCH="amd64" ;; + aarch64) ARCH="arm64" ;; + armv7l) ARCH="armv7" ;; + *) log "ERROR" "Unsupported architecture: $ARCH"; exit 1 ;; + esac + LOCAL_TGZ="ollama-linux-${ARCH}.tgz" + log "INFO" "System architecture: $ARCH" +} + +# Install system dependencies +install_dependencies() { + log "INFO" "Step 2/8: Installing system dependencies..." + local deps=(curl wget tar gzip jq) + + case "$OS_ID" in + ubuntu|debian) + if ! apt-get update; then + log "ERROR" "APT source update failed" + exit 1 + fi + if ! DEBIAN_FRONTEND=noninteractive apt-get install -y "${deps[@]}"; then + log "ERROR" "APT dependency installation failed" + exit 1 + fi + ;; + centos|rhel|fedora|openEuler|kylin|uos) + if ! yum install -y "${deps[@]}"; then + log "ERROR" "YUM dependency installation failed" + exit 1 + fi + ;; + *) + log "ERROR" "Unsupported distribution: $OS_ID" + exit 1 + ;; + esac + log "SUCCESS" "System dependencies installation completed" +} + +# Get Ollama download URL +get_ollama_url() { + echo "https://repo.oepkgs.net/openEuler/rpm/openEuler-22.03-LTS/contrib/eulercopilot/tools/$ARCH/ollama-linux-$ARCH.tgz" +} + +install_ollama() { + log "INFO" "Step 3/8: Installing Ollama core..." + local install_url=$(get_ollama_url) + local tmp_file="/tmp/ollama-${ARCH}.tgz" + # Enhanced cleanup logic + if [ -x "$OLLAMA_BIN_PATH" ] || [ -x "/usr/local/bin/ollama" ]; then + log "WARNING" "Found existing Ollama installation, version: $($OLLAMA_BIN_PATH --version)" + read -p "Reinstall? [y/N] " -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + log "WARNING" "Found existing Ollama installation, cleaning up..." + systemctl stop ollama 2>/dev/null || true + systemctl disable ollama 2>/dev/null || true + rm -rf ${SERVICE_FILE} 2>/dev/null + rm $(which ollama) 2>/dev/null + rm -rf ${OLLAMA_LIB_DIR} 2>/dev/null + rm -rf ${OLLAMA_DATA_DIR} 2>/dev/null + rm -rf /run/ollama 2>/dev/null + userdel ollama 2>/dev/null || true + groupdel ollama 2>/dev/null || true + else + return 0 + fi + fi + # Enhanced installation package handling + local actual_tgz_path="" + if [ -f "${LOCAL_DIR}/${LOCAL_TGZ}" ]; then + log "INFO" "Using local installation package: ${LOCAL_DIR}/${LOCAL_TGZ}" + actual_tgz_path="${LOCAL_DIR}/${LOCAL_TGZ}" + else + if ! check_network; then + log "ERROR" "Network unavailable and no local installation package found" + log "INFO" "Please pre-download ${LOCAL_TGZ} and place in ${LOCAL_DIR}" + exit 1 + fi + log "INFO" "Downloading installation package: ${install_url}" + if ! wget --show-progress -q -O "${tmp_file}" "${install_url}"; then + log "ERROR" "Download failed, exit code: $?" + exit 1 + fi + actual_tgz_path="${tmp_file}" + fi + + log "INFO" "Extracting files to system directory /usr..." + if ! tar -xzvf "$actual_tgz_path" -C /usr/; then + log "ERROR" "Extraction failed, possible reasons:\n1. File corrupted\n2. Insufficient disk space\n3. Permission issues" + exit 1 + fi + + chmod +x "$OLLAMA_BIN_PATH" + if [ ! -x "$OLLAMA_BIN_PATH" ]; then + log "ERROR" "Post-installation verification failed: executable file does not exist" + exit 1 + fi + log "SUCCESS" "Ollama core installation completed, version: $($OLLAMA_BIN_PATH --version || echo 'unknown')" + # New: Create compatibility symbolic link + if [ ! -L "/usr/local/bin/ollama" ]; then + ln -sf "$OLLAMA_BIN_PATH" "/usr/local/bin/ollama" + log "INFO" "Created symbolic link: /usr/local/bin/ollama → $OLLAMA_BIN_PATH" + fi + + # Set library path + echo "${OLLAMA_LIB_DIR}" > /etc/ld.so.conf.d/ollama.conf + ldconfig +} + +fix_user() { + log "INFO" "Step 4/8: Fixing user configuration..." + + # Terminate all processes using ollama user + if pgrep -u ollama >/dev/null; then + log "WARNING" "Found running ollama processes, terminating..." + pkill -9 -u ollama || true + sleep 2 + if pgrep -u ollama >/dev/null; then + log "ERROR" "Unable to terminate ollama user processes" + exit 1 + fi + fi + + # Clean up old user + if id ollama &>/dev/null; then + # Check if user is locked + if passwd -S ollama | grep -q 'L'; then + log "INFO" "Found locked ollama user, unlocking and setting random password..." + random_pass=$(openssl rand -base64 32 | tr -dc 'a-zA-Z0-9' | head -c 16) + usermod -p "$(openssl passwd -1 "$random_pass")" ollama + fi + + # Delete user and home directory + if ! userdel -r ollama; then + log "WARNING" "Unable to delete ollama user, attempting force delete..." + if ! userdel -f -r ollama; then + log "ERROR" "Force delete user failed, attempting manual cleanup..." + sed -i '/^ollama:/d' /etc/passwd /etc/shadow /etc/group + rm -rf /var/lib/ollama + log "WARNING" "Manually cleaned up ollama user information" + fi + fi + log "INFO" "Deleted old ollama user" + fi + + # Check if group exists + if getent group ollama >/dev/null; then + log "INFO" "ollama group already exists, will use existing group" + existing_group=true + else + existing_group=false + fi + + # Create system user + if ! useradd -r -g ollama -d /var/lib/ollama -s /bin/false ollama; then + log "ERROR" "User creation failed, attempting manual creation..." + + # Create group if it doesn't exist + if ! $existing_group; then + if ! groupadd -r ollama; then + log "ERROR" "Unable to create ollama group" + exit 1 + fi + fi + + # Try creating user again + if ! useradd -r -g ollama -d /var/lib/ollama -s /bin/false ollama; then + log "ERROR" "Manual user creation failed, please check:" + log "ERROR" "1. Whether /etc/passwd and /etc/group files are writable" + log "ERROR" "2. Whether there are conflicting users/groups in the system" + log "ERROR" "3. System user limits (/etc/login.defs)" + exit 1 + fi + fi + + # Create directory structure + mkdir -p /var/lib/ollama/.ollama/{models,bin} + chown -R ollama:ollama /var/lib/ollama + chmod -R 755 /var/lib/ollama + log "SUCCESS" "User configuration fix completed" +} + +fix_service() { + log "INFO" "Step 5/8: Configuring system service..." + cat > "${SERVICE_FILE}" </dev/null; then + log "ERROR" "Ollama not properly installed" + exit 1 + fi + if ! ollama list &>/dev/null; then + log "ERROR" "Service connection failed, please check:\n1.Service status: systemctl status ollama\n2.Port listening: ss -tuln | grep 11434" + exit 1 + fi + + log "SUCCESS" "Verification passed, you can perform the following operations:\n ollama list # View model list\n ollama run llama2 # Run example model" +} + +### Main execution flow ### +main() { + if [[ $EUID -ne 0 ]]; then + log "ERROR" "Please run this script with sudo" + exit 1 + fi + + detect_os + install_dependencies + echo -e "${MAGENTA}=== Starting Ollama Installation ===${RESET}" + install_ollama + fix_user + fix_service + restart_service + if final_check; then + echo -e "${MAGENTA}=== Ollama Installation Successful ===${RESET}" + else + echo -e "${MAGENTA}=== Ollama Installation Failed ===${RESET}" + fi +} + +main diff --git a/deploy/en/scripts/4-deploy-deepseek/deploy_deepseek.sh b/deploy/en/scripts/4-deploy-deepseek/deploy_deepseek.sh new file mode 100755 index 000000000..eaff68287 --- /dev/null +++ b/deploy/en/scripts/4-deploy-deepseek/deploy_deepseek.sh @@ -0,0 +1,252 @@ +#!/bin/bash +set -euo pipefail + +# Color definitions +GREEN='\033[0;32m' +RED='\033[0;31m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # Reset color + +# Configuration parameters +readonly MODEL_NAME="deepseek-llm-7b-chat" +readonly MODEL_URL="https://repo.oepkgs.net/openEuler/rpm/openEuler-22.03-LTS/contrib/eulercopilot/models/deepseek-llm-7b-chat-Q4_K_M.gguf" +readonly MODEL_FILE="deepseek-llm-7b-chat-Q4_K_M.gguf" +readonly MODEL_DIR="/home/eulercopilot/models" +readonly MODELLEFILE="Modelfile" +readonly TIMEOUT_DURATION=45 + +# Initialize working directory +readonly WORK_DIR=$(pwd) +mkdir -p "$MODEL_DIR" + +check_service() { + echo -e "${BLUE}Step 1/5: Checking service status...${NC}" + if ! systemctl is-active --quiet ollama; then + echo -e "${RED}Ollama service not running${NC}" + echo -e "${YELLOW}Please run ollama-install.sh to install service first${NC}" + exit 1 + fi + echo -e "${GREEN}Service status normal${NC}" +} + +check_network() { + local url_to_check=$(echo "$MODEL_URL" | awk -F/ '{print $1 "//" $3}') + echo -e "${BLUE}Step 2/5: Checking network connection...${NC}" + if curl --silent --head --fail --max-time 5 "$url_to_check" >/dev/null 2>&1; then + echo -e "${GREEN}Model repository accessible${NC}" + return 0 + else + echo -e "${RED}[ERROR] Unable to connect to model repository${NC}" + return 1 + fi +} + +show_progress() { + local pid=$1 + local cols=$(tput cols) + local bar_width=$((cols - 20)) + + while kill -0 "$pid" 2>/dev/null; do + local current_size=$(du -b "${MODEL_DIR}/${MODEL_FILE}" 2>/dev/null | awk '{print $1}') + local percent=$((current_size * 100 / EXPECTED_SIZE)) + [ $percent -gt 100 ] && percent=100 + + local filled=$((percent * bar_width / 100)) + local empty=$((bar_width - filled)) + + printf "\r[%-*s] %3d%%" "$bar_width" "$(printf '%0.s=' {1..$filled})$(printf '%0.s ' {1..$empty})" "$percent" + sleep 1 + done +} + +detect_gpu() { + echo -e "${BLUE}Detecting GPU devices...${NC}" + + # Check if NVIDIA GPU hardware exists + if lspci | grep -i nvidia || [ -c /dev/nvidia0 ]; then + echo -e "${GREEN}Detected NVIDIA GPU device${NC}" + + # Check if NVIDIA driver is installed + if command -v nvidia-smi &> /dev/null; then + echo -e "${GREEN}NVIDIA driver installed${NC}" + return 0 + else + echo -e "${YELLOW}GPU detected but NVIDIA driver not installed, will use CPU mode${NC}" + return 1 + fi + else + echo -e "${YELLOW}No GPU device detected, will use CPU mode${NC}" + return 1 + fi +} + +handle_model() { + echo -e "${BLUE}Step 3/5: Processing model file...${NC}" + local model_path="${MODEL_DIR}/${MODEL_FILE}" + + # Check local model + if [[ -f "$model_path" ]]; then + echo -e "${GREEN}Detected local model file ${model_path}${NC}" + return 0 + fi + + # Check network when download is needed + if ! check_network; then + echo -e "${RED}[ERROR] Unable to download model: network connection unavailable${NC}" + exit 1 + fi + + # Online download mode + echo -e "${YELLOW}Starting online model download...${NC}" + + # Get expected file size + EXPECTED_SIZE=$(curl -sLI "$MODEL_URL" | grep -i 'Content-Length' | awk '{print $2}' | tr -d '\r') + EXPECTED_SIZE=${EXPECTED_SIZE:-0} + + # Start download process + wget --tries=3 --content-disposition -O "$model_path" "$MODEL_URL" --progress=dot:binary >/dev/null 2>&1 & + local wget_pid=$! + + # Show progress bar + show_progress $wget_pid + + # Wait for download to complete + if wait $wget_pid; then + echo -e "\n${GREEN}[SUCCESS] Model download completed (file size: $(du -h "$model_path" | awk '{print $1}'))${NC}" + else + echo -e "\n${RED}[ERROR] Model download failed${NC}" + echo -e "${YELLOW}Possible reasons:" + echo "1. URL expired (current URL: $MODEL_URL)" + echo "2. Network connection issue" + echo -e "3. Insufficient disk space (current available: $(df -h ${MODEL_DIR} | awk 'NR==2 {print $4}'))${NC}" + exit 1 + fi +} + +create_modelfile() { + echo -e "${BLUE}Step 4/5: Creating model configuration...${NC}" + + # GPU parameter configuration + local gpu_param="" + if detect_gpu; then + gpu_param="PARAMETER num_gpu -1" + echo -e "${GREEN}GPU acceleration mode enabled${NC}" + else + echo -e "${YELLOW}Using CPU mode${NC}" + gpu_param="PARAMETER num_gpu 0" + fi + + cat > "${WORK_DIR}/${MODELLEFILE}" < /dev/null; then + echo -e "${YELLOW}Note: jq not installed, response parsing may be limited, recommend installing jq for better output.${NC}" + fi + + local retries=3 + local interval=5 + local attempt=1 + + if ! ollama list | grep -qw "${MODEL_NAME}"; then + echo -e "${RED}Basic verification failed - model ${MODEL_NAME} not found${NC}" + exit 1 + fi + echo -e "${GREEN}Basic verification passed - detected model: ${MODEL_NAME}${NC}" + + echo -e "${YELLOW}Executing API test (timeout ${TIMEOUT_DURATION} seconds)...${NC}" + + while [ $attempt -le $retries ]; do + echo -e "${BLUE}Attempt $attempt: Sending test request...${NC}" + + response=$(timeout ${TIMEOUT_DURATION} curl -s http://localhost:11434/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-123456" \ + -d '{ + "model": "'"${MODEL_NAME}"'", + "messages": [ + {"role": "system", "content": "You are an AI assistant"}, + {"role": "user", "content": "Hello, please recite a Chinese ancient poem"} + ], + "stream": false + }') + + if [[ $? -eq 0 ]] && [[ -n "$response" ]]; then + echo -e "${GREEN}Test response successful, received valid output:" + if jq -e .choices[0].message.content <<< "$response" &> /dev/null; then + jq .choices[0].message.content <<< "$response" + else + echo "$response" + fi + return 0 + else + echo -e "${YELLOW}Request did not receive valid response, retrying...${NC}" + ((attempt++)) + sleep $interval + fi + done + + echo -e "${RED}Verification failed: No valid response received after ${retries} retries${NC}" + return 1 +} + +### Main execution flow ### +echo -e "${BLUE}=== Starting Model Deployment ===${NC}" +{ + check_service + handle_model + create_modelfile + create_model + verify_deployment || true +} + +echo -e "${GREEN}=== Model Deployment Successful ===${NC}" + +echo -e "${YELLOW}Usage Instructions:${NC}" +echo -e "${YELLOW}1. Start interactive mode: ollama run $MODEL_NAME${NC}" +echo -e "${BLUE}2. API access example:${NC}" +cat << EOF +curl http://localhost:11434/v1/chat/completions \\ +-H "Content-Type: application/json" \\ +-H "Authorization: Bearer sk-123456" \\ +-d '{ + "model": "$MODEL_NAME", + "messages": [ + {"role": "system", "content": "You are an AI assistant"}, + {"role": "user", "content": "Hello"} + ], + "stream": false, + "n": 1, + "max_tokens": 2048 +}' +EOF + +echo -e "${BLUE}3. Streaming conversation mode:${NC}" +echo -e "${BLUE}Change "stream": false to "stream": true in the above request${NC}" diff --git a/deploy/en/scripts/5-deploy-embedding/deploy-embedding.sh b/deploy/en/scripts/5-deploy-embedding/deploy-embedding.sh new file mode 100755 index 000000000..3158c95d9 --- /dev/null +++ b/deploy/en/scripts/5-deploy-embedding/deploy-embedding.sh @@ -0,0 +1,252 @@ +#!/bin/bash +set -euo pipefail + +# Color definitions +RED='\e[31m' +GREEN='\e[32m' +YELLOW='\e[33m' +BLUE='\e[34m' +NC='\e[0m' # Reset color + +# Configuration parameters +readonly MODEL_NAME="bge-m3" +readonly MODEL_URL="https://modelscope.cn/models/gpustack/bge-m3-GGUF/resolve/master/bge-m3-Q4_K_M.gguf" +readonly MODEL_FILE="bge-m3-Q4_K_M.gguf" +readonly MODELLEFILE="Modelfile" +readonly TIMEOUT_DURATION=45 +readonly MODEL_DIR="/home/eulercopilot/models" + +# Initialize working directory +readonly WORK_DIR=$(pwd) +mkdir -p "$MODEL_DIR" + +# Network check function (unchanged) +check_network() { + echo -e "${BLUE}Step 1/5: Checking network connection...${NC}" + local test_url="https://modelscope.cn" + if curl --silent --head --fail --max-time 5 "$test_url" >/dev/null 2>&1; then + echo -e "${GREEN}[SUCCESS] Network connection normal${NC}" + return 0 + else + echo -e "${YELLOW}[WARNING] Unable to connect to network, switching to offline mode${NC}" + return 1 + fi +} + +# Service check (unchanged) +check_service() { + echo -e "${BLUE}Step 2/5: Checking service status...${NC}" + if ! systemctl is-active --quiet ollama; then + echo -e "${RED}[ERROR] Ollama service not running${NC}" + echo -e "${YELLOW}Possible reasons:" + echo "1. Service not installed" + echo "2. System service not enabled" + echo -e "Please run ollama-install.sh to install service first${NC}" + exit 1 + fi +} + +show_progress() { + local pid=$1 + local cols=$(tput cols) + local bar_width=$((cols - 20)) + + while kill -0 "$pid" 2>/dev/null; do + local current_size=$(du -b "${MODEL_DIR}/${MODEL_FILE}" 2>/dev/null | awk '{print $1}') + local percent=$((current_size * 100 / EXPECTED_SIZE)) + [ $percent -gt 100 ] && percent=100 + + local filled=$((percent * bar_width / 100)) + local empty=$((bar_width - filled)) + + printf "\r[%-*s] %3d%%" "$bar_width" "$(printf '%0.s=' {1..$filled})$(printf '%0.s ' {1..$empty})" "$percent" + sleep 1 + done +} + +detect_gpu() { + echo -e "${BLUE}Detecting GPU devices...${NC}" + + # Check if NVIDIA GPU hardware exists + if lspci | grep -i nvidia || [ -c /dev/nvidia0 ]; then + echo -e "${GREEN}Detected NVIDIA GPU device${NC}" + + # Check if NVIDIA driver is installed + if command -v nvidia-smi &> /dev/null; then + echo -e "${GREEN}NVIDIA driver installed${NC}" + return 0 + else + echo -e "${YELLOW}GPU detected but NVIDIA driver not installed, will use CPU mode${NC}" + return 1 + fi + else + echo -e "${YELLOW}No GPU device detected, will use CPU mode${NC}" + return 1 + fi +} + +handle_model() { + echo -e "${BLUE}Step 3/5: Processing model file...${NC}" + local model_path="${MODEL_DIR}/${MODEL_FILE}" + + # Check local model + if [[ -f "$model_path" ]]; then + echo -e "${GREEN}Detected local model file ${model_path}${NC}" + return 0 + fi + + # Check network when download is needed + if ! check_network; then + echo -e "${RED}[ERROR] Unable to download model: network connection unavailable${NC}" + echo -e "${YELLOW}Solutions:" + echo -e "1. Please check network connection" + echo -e "2. You can manually place model file at: ${MODEL_DIR}" + exit 1 + fi + + # Online download mode + echo -e "${YELLOW}Starting online model download...${NC}" + echo -e "${YELLOW}Download URL: ${MODEL_URL}${NC}" + + # Create temporary file to record wget output + local wget_output=$(mktemp) + + # Execute download and show dynamic progress bar + ( + wget --tries=3 --content-disposition -O "$model_path" "$MODEL_URL" --progress=dot:binary 2>&1 | \ + while IFS= read -r line; do + # Extract percentage + if [[ "$line" =~ ([0-9]{1,3})% ]]; then + local percent=${BASH_REMATCH[1]} + # Calculate progress bar length (based on terminal width - 20) + local cols=$(tput cols) + local bar_width=$((cols - 20)) + local filled=$((percent * bar_width / 100)) + local empty=$((bar_width - filled)) + + # Build progress bar + local progress_bar=$(printf "%${filled}s" | tr ' ' '=') + local remaining_bar=$(printf "%${empty}s" | tr ' ' ' ') + + # Show progress (use carriage return to overwrite) + printf "\r[%s%s] %3d%%" "$progress_bar" "$remaining_bar" "$percent" + fi + done + echo # New line + ) | tee "$wget_output" + + # Check download result + if grep -q "100%" "$wget_output"; then + echo -e "${GREEN}[SUCCESS] Model download completed (file size: $(du -h "$model_path" | awk '{print $1}'))${NC}" + echo -e "${GREEN}Storage path: ${model_path}${NC}" + rm -f "$wget_output" + else + echo -e "${RED}[ERROR] Model download failed${NC}" + echo -e "${YELLOW}Possible reasons:" + echo "1. URL expired (current URL: $MODEL_URL)" + echo "2. Network connection issue" + echo -e "3. Insufficient disk space (current available: $(df -h ${MODEL_DIR} | awk 'NR==2 {print $4}'))${NC}" + rm -f "$wget_output" + exit 1 + fi +} + +create_modelfile() { + echo -e "${BLUE}Step 4/5: Creating model configuration...${NC}" + + # GPU parameter configuration + local gpu_param="" + if detect_gpu; then + gpu_param="PARAMETER num_gpu -1" + echo -e "${GREEN}GPU acceleration mode enabled${NC}" + else + echo -e "${YELLOW}Using CPU mode${NC}" + gpu_param="PARAMETER num_gpu 0" + fi + + cat > "${WORK_DIR}/${MODELLEFILE}" </dev/null 2>&1 + pwd +)/$(basename "${BASH_SOURCE[0]}")" + +CHART_DIR="$( + canonical_path=$(readlink -f "$SCRIPT_PATH" 2>/dev/null || echo "$SCRIPT_PATH") + dirname "$(dirname "$(dirname "$canonical_path")")" +)/chart" + +# Get system architecture +get_architecture() { + arch=$(uname -m) + case "$arch" in + x86_64) + arch="x86" + ;; + aarch64) + arch="arm" + ;; + *) + echo -e "${RED}Error: Unsupported architecture $arch${NC}" + exit 1 + ;; + esac + echo -e "${GREEN}Detected system architecture: $(uname -m)${NC}" +} + +create_namespace() { + echo -e "${BLUE}==> Checking namespace euler-copilot...${NC}" + if ! kubectl get namespace euler-copilot &> /dev/null; then + kubectl create namespace euler-copilot || { + echo -e "${RED}Namespace creation failed!${NC}" + return 1 + } + echo -e "${GREEN}Namespace created successfully${NC}" + else + echo -e "${YELLOW}Namespace already exists, skipping creation${NC}" + fi +} + +uninstall_databases() { + echo -e "${BLUE}==> Cleaning up existing resources...${NC}" + + local helm_releases + helm_releases=$(helm list -n euler-copilot -q --filter '^databases' 2>/dev/null || true) + + if [ -n "$helm_releases" ]; then + echo -e "${YELLOW}Found the following Helm Releases, starting cleanup...${NC}" + while IFS= read -r release; do + echo -e "${BLUE}Deleting Helm Release: ${release}${NC}" + if ! helm uninstall "$release" -n euler-copilot --wait --timeout 2m; then + echo -e "${RED}Error: Failed to delete Helm Release ${release}!${NC}" >&2 + return 1 + fi + done <<< "$helm_releases" + else + echo -e "${YELLOW}No Helm Releases found to clean${NC}" + fi + + # Modified: Only filter specific PVC names + local pvc_list + pvc_list=$(kubectl get pvc -n euler-copilot -o jsonpath='{.items[*].metadata.name}' 2>/dev/null \ + | tr ' ' '\n' \ + | grep -E '^(opengauss-storage|mongo-storage|minio-storage)$' || true) # Exact match for three specified names + + if [ -n "$pvc_list" ]; then + echo -e "${YELLOW}Found the following PVCs, starting cleanup...${NC}" + while IFS= read -r pvc; do + echo -e "${BLUE}Deleting PVC: $pvc${NC}" + if ! kubectl delete pvc "$pvc" -n euler-copilot --force --grace-period=0; then + echo -e "${RED}Error: Failed to delete PVC $pvc!${NC}" >&2 + return 1 + fi + done <<< "$pvc_list" + else + echo -e "${YELLOW}No PVCs found to clean${NC}" + fi + + # New: Delete euler-copilot-database Secret + local secret_name="euler-copilot-database" + if kubectl get secret "$secret_name" -n euler-copilot &>/dev/null; then + echo -e "${YELLOW}Found Secret: ${secret_name}, starting cleanup...${NC}" + if ! kubectl delete secret "$secret_name" -n euler-copilot; then + echo -e "${RED}Error: Failed to delete Secret ${secret_name}!${NC}" >&2 + return 1 + fi + else + echo -e "${YELLOW}No Secret found to clean: ${secret_name}${NC}" + fi + + echo -e "${BLUE}Waiting for resource cleanup to complete (10 seconds)...${NC}" + sleep 10 + + echo -e "${GREEN}Resource cleanup completed${NC}" +} + +helm_install() { + echo -e "${BLUE}==> Entering deployment directory...${NC}" + [ ! -d "$CHART_DIR" ] && { + echo -e "${RED}Error: Deployment directory does not exist $CHART_DIR${NC}" + return 1 + } + cd "$CHART_DIR" + + echo -e "${BLUE}Installing databases...${NC}" + helm upgrade --install databases --set globals.arch=$arch -n euler-copilot ./databases || { + echo -e "${RED}Helm installation of databases failed!${NC}" + return 1 + } +} + +check_pods_status() { + echo -e "${BLUE}==> Waiting for initialization to complete (30 seconds)...${NC}" + sleep 30 + + local timeout=300 + local start_time=$(date +%s) + + echo -e "${BLUE}Starting pod status monitoring (total timeout 300 seconds)...${NC}" + + while true; do + local current_time=$(date +%s) + local elapsed=$((current_time - start_time)) + + if [ $elapsed -gt $timeout ]; then + echo -e "${RED}Error: Deployment timeout!${NC}" + kubectl get pods -n euler-copilot + return 1 + fi + + local not_running=$(kubectl get pods -n euler-copilot -o jsonpath='{range .items[*]}{.metadata.name} {.status.phase}{"\n"}{end}' | grep -v "Running") + + if [ -z "$not_running" ]; then + echo -e "${GREEN}All pods are running normally!${NC}" + kubectl get pods -n euler-copilot + return 0 + else + echo "Waiting for pods to be ready (waited ${elapsed} seconds)..." + echo "Current abnormal pods:" + echo "$not_running" | awk '{print " - " $1 " (" $2 ")"}' + sleep 10 + fi + done +} + +main() { + get_architecture + create_namespace + uninstall_databases + helm_install + check_pods_status + + echo -e "\n${GREEN}=========================" + echo "Database deployment completed!" + echo -e "=========================${NC}" +} + +trap 'echo -e "${RED}Operation interrupted!${NC}"; exit 1' INT +main "$@" diff --git a/deploy/en/scripts/7-install-authhub/install_authhub.sh b/deploy/en/scripts/7-install-authhub/install_authhub.sh new file mode 100755 index 000000000..9ccfb805d --- /dev/null +++ b/deploy/en/scripts/7-install-authhub/install_authhub.sh @@ -0,0 +1,240 @@ +#!/bin/bash + +set -eo pipefail + +RED='\033[31m' +GREEN='\033[32m' +YELLOW='\033[33m' +BLUE='\033[34m' +NC='\033[0m' + +SCRIPT_PATH="$( + cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 + pwd +)/$(basename "${BASH_SOURCE[0]}")" + +CHART_DIR="$( + canonical_path=$(readlink -f "$SCRIPT_PATH" 2>/dev/null || echo "$SCRIPT_PATH") + dirname "$(dirname "$(dirname "$canonical_path")")" +)/chart" + +# Print help information +print_help() { + echo -e "${GREEN}Usage: $0 [options]" + echo -e "Options:" + echo -e " --help Show help information" + echo -e " --authhub_address
Specify Authhub access address (e.g.: http://myhost:30081)" + echo -e "" + echo -e "Example:" + echo -e " $0 --authhub_address http://myhost:30081${NC}" + exit 0 +} + +# Get system architecture +get_architecture() { + local arch=$(uname -m) + case "$arch" in + x86_64) + arch="x86" + ;; + aarch64) + arch="arm" + ;; + *) + echo -e "${RED}Error: Unsupported architecture $arch${NC}" >&2 + return 1 + ;; + esac + echo -e "${GREEN}Detected system architecture: $(uname -m)${NC}" >&2 + echo "$arch" +} + +create_namespace() { + echo -e "${BLUE}==> Checking namespace euler-copilot...${NC}" + if ! kubectl get namespace euler-copilot &> /dev/null; then + kubectl create namespace euler-copilot || { + echo -e "${RED}Namespace creation failed!${NC}" + return 1 + } + echo -e "${GREEN}Namespace created successfully${NC}" + else + echo -e "${YELLOW}Namespace already exists, skipping creation${NC}" + fi +} + +uninstall_authhub() { + echo -e "${BLUE}==> Cleaning up existing resources...${NC}" + + local RELEASES + RELEASES=$(helm list -n euler-copilot --short | grep authhub || true) + + if [ -n "$RELEASES" ]; then + echo -e "${YELLOW}Found Helm Releases, starting cleanup...${NC}" + for release in $RELEASES; do + echo -e "${BLUE}Deleting Helm Release: ${release}${NC}" + helm uninstall "$release" -n euler-copilot || echo -e "${RED}Failed to delete Helm Release, continuing...${NC}" + done + else + echo -e "${YELLOW}No Helm Releases found to clean${NC}" + fi + + local pvc_name + pvc_name=$(kubectl get pvc -n euler-copilot | grep 'mysql-pvc' 2>/dev/null || true) + + if [ -n "$pvc_name" ]; then + echo -e "${YELLOW}Found PVCs, starting cleanup...${NC}" + kubectl delete pvc mysql-pvc -n euler-copilot --force --grace-period=0 || echo -e "${RED}PVC deletion failed, continuing...${NC}" + else + echo -e "${YELLOW}No PVCs found to clean${NC}" + fi + + # New: Delete authhub-secret + local authhub_secret="authhub-secret" + if kubectl get secret "$authhub_secret" -n euler-copilot &>/dev/null; then + echo -e "${YELLOW}Found Secret: ${authhub_secret}, starting cleanup...${NC}" + if ! kubectl delete secret "$authhub_secret" -n euler-copilot; then + echo -e "${RED}Error: Failed to delete Secret ${authhub_secret}!${NC}" >&2 + return 1 + fi + else + echo -e "${YELLOW}No Secret found to clean: ${authhub_secret}${NC}" + fi + + echo -e "${GREEN}Resource cleanup completed${NC}" +} + +get_authhub_address() { + local default_address="http://127.0.0.1:30081" + + echo -e "${BLUE}Enter Authhub access address (IP or domain)${NC}" + read -p "Authhub address: " authhub_address + + # Handle empty input + if [[ -z "$authhub_address" ]]; then + authhub_address="$default_address" + echo -e "${GREEN}Using default address: ${authhub_address}${NC}" + else + echo -e "${GREEN}Input address: ${authhub_address}${NC}" + fi + + return 0 +} + +helm_install() { + local arch="$1" + echo -e "${BLUE}==> Entering deployment directory...${NC}" + [ ! -d "${CHART_DIR}" ] && { + echo -e "${RED}Error: Deployment directory does not exist ${CHART_DIR} ${NC}" + return 1 + } + cd "${CHART_DIR}" + + echo -e "${BLUE}Installing authhub...${NC}" + helm upgrade --install authhub -n euler-copilot ./authhub \ + --set globals.arch="$arch" \ + --set domain.authhub="${authhub_address}" || { + echo -e "${RED}Helm installation of authhub failed!${NC}" + return 1 + } +} + +check_pods_status() { + echo -e "${BLUE}==> Waiting for initialization (30 seconds)...${NC}" >&2 + sleep 30 + + local timeout=300 + local start_time=$(date +%s) + + echo -e "${BLUE}Monitoring pod status (total timeout 300 seconds)...${NC}" >&2 + + while true; do + local current_time=$(date +%s) + local elapsed=$((current_time - start_time)) + + if [ $elapsed -gt $timeout ]; then + echo -e "${YELLOW}Warning: Deployment timeout! Please check the following resources:${NC}" >&2 + kubectl get pods -n euler-copilot -o wide + echo -e "\n${YELLOW}Suggestions:${NC}" + echo "1. Check logs of unready pods: kubectl logs -n euler-copilot " + echo "2. Check PVC status: kubectl get pvc -n euler-copilot" + echo "3. Check Service status: kubectl get svc -n euler-copilot" + return 1 + fi + + local not_running=$(kubectl get pods -n euler-copilot -o jsonpath='{range .items[*]}{.metadata.name} {.status.phase} {.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' \ + | awk '$2 != "Running" || $3 != "True" {print $1 " " $2}') + + if [ -z "$not_running" ]; then + echo -e "${GREEN}All pods are running normally!${NC}" >&2 + kubectl get pods -n euler-copilot -o wide + return 0 + else + echo "Waiting for pods to be ready (waited ${elapsed} seconds)..." + echo "Current unready pods:" + echo "$not_running" | awk '{print " - " $1 " (" $2 ")"}' + sleep 10 + fi + done +} + +deploy() { + local arch + arch=$(get_architecture) || exit 1 + create_namespace || exit 1 + uninstall_authhub || exit 1 + + # If address not provided via parameter, prompt user + if [ -z "$authhub_address" ]; then + echo -e "${YELLOW}--authhub_address parameter not provided, need manual input${NC}" + get_authhub_address || exit 1 + else + echo -e "${GREEN}Using parameter-specified Authhub address: $authhub_address${NC}" + fi + + helm_install "$arch" || exit 1 + check_pods_status || { + echo -e "${RED}Deployment failed: Pod status check not passed!${NC}" + exit 1 + } + + echo -e "\n${GREEN}=========================" + echo -e "Authhub deployment completed!" + echo -e "Check pod status: kubectl get pod -n euler-copilot" + echo -e "Authhub login address: $authhub_address" + echo -e "Default credentials: administrator/changeme" + echo -e "=========================${NC}" +} + +# Parse command line arguments +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --help) + print_help + exit 0 + ;; + --authhub_address) + if [ -n "$2" ]; then + authhub_address="$2" + shift 2 + else + echo -e "${RED}Error: --authhub_address requires a parameter${NC}" >&2 + exit 1 + fi + ;; + *) + echo -e "${RED}Unknown parameter: $1${NC}" >&2 + print_help + exit 1 + ;; + esac + done +} + +main() { + parse_args "$@" + deploy +} + +trap 'echo -e "${RED}Operation interrupted!${NC}"; exit 1' INT +main "$@" diff --git a/deploy/en/scripts/8-install-EulerCopilot/install_eulercopilot.sh b/deploy/en/scripts/8-install-EulerCopilot/install_eulercopilot.sh new file mode 100755 index 000000000..5095d1f77 --- /dev/null +++ b/deploy/en/scripts/8-install-EulerCopilot/install_eulercopilot.sh @@ -0,0 +1,480 @@ +#!/bin/bash + +set -eo pipefail + +# Color definitions +RED='\e[31m' +GREEN='\e[32m' +YELLOW='\e[33m' +BLUE='\e[34m' +NC='\e[0m' # Reset color + +NAMESPACE="euler-copilot" +PLUGINS_DIR="/var/lib/eulercopilot" + +# Global variables +client_id="" +client_secret="" +eulercopilot_address="" +authhub_address="" + +SCRIPT_PATH="$( + cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 + pwd +)/$(basename "${BASH_SOURCE[0]}")" + +DEPLOY_DIR="$( + canonical_path=$(readlink -f "$SCRIPT_PATH" 2>/dev/null || echo "$SCRIPT_PATH") + dirname "$(dirname "$(dirname "$canonical_path")")" +)" + +# Show help information +show_help() { + echo -e "${GREEN}Usage: $0 [options]" + echo -e "Options:" + echo -e " --help Show this help message" + echo -e " --eulercopilot_address Specify EulerCopilot frontend access URL" + echo -e " --authhub_address Specify Authhub frontend access URL" + echo -e "" + echo -e "Example:" + echo -e " $0 --eulercopilot_address http://myhost:30080 --authhub_address http://myhost:30081${NC}" + exit 0 +} + +# Parse command line arguments +parse_arguments() { + while [[ $# -gt 0 ]]; do + case "$1" in + --help) + show_help + ;; + --eulercopilot_address) + if [ -n "$2" ]; then + eulercopilot_address="$2" + shift + else + echo -e "${RED}Error: --eulercopilot_address requires a value${NC}" >&2 + exit 1 + fi + ;; + --authhub_address) + if [ -n "$2" ]; then + authhub_address="$2" + shift + else + echo -e "${RED}Error: --authhub_address requires a value${NC}" >&2 + exit 1 + fi + ;; + *) + echo -e "${RED}Unknown option: $1${NC}" >&2 + show_help + exit 1 + ;; + esac + shift + done +} + +# Get system architecture +get_architecture() { + local arch=$(uname -m) + case "$arch" in + x86_64) arch="x86" ;; + aarch64) arch="arm" ;; + *) + echo -e "${RED}Error: Unsupported architecture $arch${NC}" >&2 + return 1 + ;; + esac + echo -e "${GREEN}Detected system architecture: ${arch} (original: $(uname -m))${NC}" >&2 + echo "$arch" +} + +# Auto-detect business network interface +get_network_ip() { + echo -e "${BLUE}Auto-detecting business network interface IP address...${NC}" >&2 + local timeout=20 + local start_time=$(date +%s) + local interface="" + local host="" + + # Find available network interfaces + while [ $(( $(date +%s) - start_time )) -lt $timeout ]; do + # Get all non-virtual interfaces (exclude lo, docker, veth, etc.) + interfaces=$(ip -o link show | awk -F': ' '{print $2}' | grep -vE '^lo$|docker|veth|br-|virbr|tun') + + for intf in $interfaces; do + # Check if interface status is UP + if ip link show "$intf" | grep -q 'state UP'; then + # Get IPv4 address + ip_addr=$(ip addr show "$intf" | grep -w inet | awk '{print $2}' | cut -d'/' -f1) + if [ -n "$ip_addr" ]; then + interface=$intf + host=$ip_addr + break 2 # Break two levels + fi + fi + done + sleep 1 + done + + if [ -z "$interface" ]; then + echo -e "${RED}Error: No available business network interface found${NC}" >&2 + exit 1 + fi + + echo -e "${GREEN}Using network interface: ${interface}, IP address: ${host}${NC}" >&2 + echo "$host" +} + +get_address_input() { + # If addresses already provided via command line, use them directly + if [ -n "$eulercopilot_address" ] && [ -n "$authhub_address" ]; then + echo -e "${GREEN}Using command line parameters:" + echo "EulerCopilot address: $eulercopilot_address" + echo "Authhub address: $authhub_address" + return + fi + + # Read from environment variables or use defaults + eulercopilot_address=${EULERCOPILOT_ADDRESS:-"http://127.0.0.1:30080"} + authhub_address=${AUTHHUB_ADDRESS:-"http://127.0.0.1:30081"} + + # Non-interactive mode uses defaults directly + if [ -t 0 ]; then # Only show prompts in interactive terminal + echo -e "${BLUE}Enter EulerCopilot frontend access URL (default: $eulercopilot_address):${NC}" + read -p "> " input_euler + [ -n "$input_euler" ] && eulercopilot_address=$input_euler + + echo -e "${BLUE}Enter Authhub frontend access URL (default: $authhub_address):${NC}" + read -p "> " input_auth + [ -n "$input_auth" ] && authhub_address=$input_auth + fi + + echo -e "${GREEN}Using configuration:" + echo "EulerCopilot address: $eulercopilot_address" + echo "Authhub address: $authhub_address" +} + +get_client_info_auto() { + # Get user input addresses + get_address_input + # Create temporary file + local temp_file + temp_file=$(mktemp) + + # Directly call Python script and pass domain parameters + python3 "${DEPLOY_DIR}/scripts/9-other-script/get_client_id_and_secret.py" "${eulercopilot_address}" > "$temp_file" 2>&1 + + # Check Python script execution result + if [ $? -ne 0 ]; then + echo -e "${RED}Error: Python script execution failed${NC}" + cat "$temp_file" + rm -f "$temp_file" + return 1 + fi + + # Extract credential information + client_id=$(grep "client_id: " "$temp_file" | awk '{print $2}') + client_secret=$(grep "client_secret: " "$temp_file" | awk '{print $2}') + rm -f "$temp_file" + + # Verify results + if [ -z "$client_id" ] || [ -z "$client_secret" ]; then + echo -e "${RED}Error: Unable to get valid client credentials${NC}" >&2 + return 1 + fi + + # Output results + echo -e "${GREEN}==============================${NC}" + echo -e "${GREEN}Client ID: ${client_id}${NC}" + echo -e "${GREEN}Client Secret: ${client_secret}${NC}" + echo -e "${GREEN}==============================${NC}" +} + +get_client_info_manual() { + # Non-interactive mode uses defaults directly + if [ -t 0 ]; then # Only show prompts in interactive terminal + echo -e "${BLUE}Enter Client ID: domain (endpoint info: Client ID): ${NC}" + read -p "> " input_id + [ -n "$input_id" ] && client_id=$input_id + + echo -e "${BLUE}Enter Client Secret: domain (endpoint info: Client Secret):${NC}" + read -p "> " input_secret + [ -n "$input_secret" ] && client_secret=$input_secret + fi + + # Unified domain format verification + echo -e "${GREEN}Using configuration:" + echo "Client ID: $client_id" + echo "Client Secret: $client_secret" +} + +check_directories() { + echo -e "${BLUE}Checking if semantic interface directory exists...${NC}" >&2 + + # Define parent directory and subdirectory list + local REQUIRED_OWNER="root:root" + + # Check and create parent directory + if [ -d "${PLUGINS_DIR}" ]; then + echo -e "${GREEN}Directory exists: ${PLUGINS_DIR}${NC}" >&2 + # Check current permissions + local current_owner=$(stat -c "%u:%g" "${PLUGINS_DIR}" 2>/dev/null) + if [ "$current_owner" != "$REQUIRED_OWNER" ]; then + echo -e "${YELLOW}Current directory permissions: ${current_owner}, modifying to ${REQUIRED_OWNER}...${NC}" >&2 + if chown root:root "${PLUGINS_DIR}"; then + echo -e "${GREEN}Directory permissions successfully modified to ${REQUIRED_OWNER}${NC}" >&2 + else + echo -e "${RED}Error: Unable to modify directory permissions to ${REQUIRED_OWNER}${NC}" >&2 + exit 1 + fi + else + echo -e "${GREEN}Directory permissions correct (${REQUIRED_OWNER})${NC}" >&2 + fi + else + if mkdir -p "${PLUGINS_DIR}"; then + echo -e "${GREEN}Directory created: ${PLUGINS_DIR}${NC}" >&2 + chown root:root "${PLUGINS_DIR}" # Set parent directory owner + else + echo -e "${RED}Error: Unable to create directory ${PLUGINS_DIR}${NC}" >&2 + exit 1 + fi + fi +} + +uninstall_eulercopilot() { + echo -e "${YELLOW}Checking if EulerCopilot is already deployed...${NC}" >&2 + + # Delete Helm Release: euler-copilot + if helm list -n euler-copilot --short | grep -q '^euler-copilot$'; then + echo -e "${GREEN}Found Helm Release: euler-copilot, starting cleanup...${NC}" + if ! helm uninstall euler-copilot -n euler-copilot; then + echo -e "${RED}Error: Failed to delete Helm Release euler-copilot!${NC}" >&2 + return 1 + fi + else + echo -e "${YELLOW}No Helm Release found to clean: euler-copilot${NC}" + fi + + # Delete PVC: framework-semantics-claim + local pvc_name="framework-semantics-claim" + if kubectl get pvc "$pvc_name" -n euler-copilot &>/dev/null; then + echo -e "${GREEN}Found PVC: ${pvc_name}, starting cleanup...${NC}" + if ! kubectl delete pvc "$pvc_name" -n euler-copilot --force --grace-period=0; then + echo -e "${RED}Error: Failed to delete PVC ${pvc_name}!${NC}" >&2 + return 1 + fi + else + echo -e "${YELLOW}No PVC found to clean: ${pvc_name}${NC}" + fi + + # Delete Secret: euler-copilot-system + local secret_name="euler-copilot-system" + if kubectl get secret "$secret_name" -n euler-copilot &>/dev/null; then + echo -e "${GREEN}Found Secret: ${secret_name}, starting cleanup...${NC}" + if ! kubectl delete secret "$secret_name" -n euler-copilot; then + echo -e "${RED}Error: Failed to delete Secret ${secret_name}!${NC}" >&2 + return 1 + fi + else + echo -e "${YELLOW}No Secret found to clean: ${secret_name}${NC}" + fi + + echo -e "${GREEN}Resource cleanup completed${NC}" +} + +modify_yaml() { + local host=$1 + local preserve_models=$2 # New parameter, indicates whether to preserve model configuration + echo -e "${BLUE}Starting YAML configuration file modification...${NC}" >&2 + + # Build parameter array + local set_args=() + + # Add other required parameters + set_args+=( + "--set" "login.client.id=${client_id}" + "--set" "login.client.secret=${client_secret}" + "--set" "domain.euler_copilot=${eulercopilot_address}" + "--set" "domain.authhub=${authhub_address}" + ) + + # If model configuration doesn't need to be preserved, add model-related parameters + if [[ "$preserve_models" != [Yy]* ]]; then + set_args+=( + "--set" "models.answer.endpoint=http://$host:11434/v1" + "--set" "models.answer.key=sk-123456" + "--set" "models.answer.name=deepseek-llm-7b-chat:latest" + "--set" "models.functionCall.backend=ollama" + "--set" "models.functionCall.endpoint=http://$host:11434" + "--set" "models.embedding.type=openai" + "--set" "models.embedding.endpoint=http://$host:11434/v1" + "--set" "models.embedding.key=sk-123456" + "--set" "models.embedding.name=bge-m3:latest" + ) + fi + + # Call Python script, pass all parameters + python3 "${DEPLOY_DIR}/scripts/9-other-script/modify_eulercopilot_yaml.py" \ + "${DEPLOY_DIR}/chart/euler_copilot/values.yaml" \ + "${DEPLOY_DIR}/chart/euler_copilot/values.yaml" \ + "${set_args[@]}" || { + echo -e "${RED}Error: YAML file modification failed${NC}" >&2 + exit 1 + } + echo -e "${GREEN}YAML file modified successfully!${NC}" >&2 +} + +# Check directory +enter_chart_directory() { + echo -e "${BLUE}Entering Chart directory...${NC}" >&2 + cd "${DEPLOY_DIR}/chart/" || { + echo -e "${RED}Error: Unable to enter Chart directory ${DEPLOY_DIR}/chart/${NC}" >&2 + exit 1 + } +} + +pre_install_checks() { + # Check if kubectl and helm are available + command -v kubectl >/dev/null 2>&1 || error_exit "kubectl not installed" + command -v helm >/dev/null 2>&1 || error_exit "helm not installed" + + # Check Kubernetes cluster connection + kubectl cluster-info >/dev/null 2>&1 || error_exit "Unable to connect to Kubernetes cluster" + + # Check necessary storage classes + kubectl get storageclasses >/dev/null 2>&1 || error_exit "Unable to get storage class information" +} + +# Execute installation +execute_helm_install() { + local arch=$1 + echo -e "${BLUE}Starting EulerCopilot deployment (architecture: $arch)...${NC}" >&2 + + enter_chart_directory + helm upgrade --install $NAMESPACE -n $NAMESPACE ./euler_copilot --set globals.arch=$arch --create-namespace || { + echo -e "${RED}Helm installation of EulerCopilot failed!${NC}" >&2 + exit 1 + } + echo -e "${GREEN}Helm installation of EulerCopilot successful!${NC}" >&2 +} + +# Check pod status +check_pods_status() { + echo -e "${BLUE}==> Waiting for initialization (30 seconds)...${NC}" >&2 + sleep 30 + + local timeout=300 + local start_time=$(date +%s) + + echo -e "${BLUE}Starting pod status monitoring (total timeout 300 seconds)...${NC}" >&2 + + while true; do + local current_time=$(date +%s) + local elapsed=$((current_time - start_time)) + + if [ $elapsed -gt $timeout ]; then + echo -e "${YELLOW}Warning: Deployment timeout! Please check the following resources:${NC}" >&2 + kubectl get pods -n $NAMESPACE -o wide + echo -e "\n${YELLOW}Suggested checks:${NC}" + echo "1. Check logs of unready pods: kubectl logs -n $NAMESPACE " + echo "2. Check PVC status: kubectl get pvc -n $NAMESPACE" + echo "3. Check Service status: kubectl get svc -n $NAMESPACE" + return 1 + fi + + local not_running=$(kubectl get pods -n $NAMESPACE -o jsonpath='{range .items[*]}{.metadata.name} {.status.phase} {.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' \ + | awk '$2 != "Running" || $3 != "True" {print $1 " " $2}') + + if [ -z "$not_running" ]; then + echo -e "${GREEN}All pods are running normally!${NC}" >&2 + kubectl get pods -n $NAMESPACE -o wide + return 0 + else + echo "Waiting for pods to be ready (waited ${elapsed} seconds)..." + echo "Current unready pods:" + echo "$not_running" | awk '{print " - " $1 " (" $2 ")"}' + sleep 10 + fi + done +} + +# Modify main function +main() { + parse_arguments "$@" + + pre_install_checks + + local arch host + arch=$(get_architecture) || exit 1 + host=$(get_network_ip) || exit 1 + + uninstall_eulercopilot + + if ! get_client_info_auto; then + get_client_info_manual + fi + + check_directories + + # Interactive prompt optimization + if [ -t 0 ]; then + echo -e "${YELLOW}Keep existing model configuration?${NC}" + echo -e " ${BLUE}Y) Keep existing configuration${NC}" + echo -e " ${BLUE}n) Use default configuration${NC}" + while true; do + read -p "Please choose (Y/N): " input_preserve + case "${input_preserve:-Y}" in + [YyNn]) preserve_models=${input_preserve:-Y}; break ;; + *) echo -e "${RED}Invalid input, please choose Y or n${NC}" ;; + esac + done + else + preserve_models="N" + fi + + echo -e "${BLUE}Starting YAML configuration modification...${NC}" + modify_yaml "$host" "$preserve_models" + + echo -e "${BLUE}Starting Helm installation...${NC}" + execute_helm_install "$arch" + + if check_pods_status; then + echo -e "${GREEN}All components are ready!${NC}" + show_success_message "$host" "$arch" + else + echo -e "${YELLOW}Some components are not ready, recommended to investigate!${NC}" + fi +} + +# Add installation success message display function +show_success_message() { + local host=$1 + local arch=$2 + + echo -e "\n${GREEN}==================================================${NC}" + echo -e "${GREEN} EulerCopilot Deployment Completed! ${NC}" + echo -e "${GREEN}==================================================${NC}" + + echo -e "${YELLOW}Access Information:${NC}" + echo -e "EulerCopilot UI: ${eulercopilot_address}" + echo -e "AuthHub Admin UI: ${authhub_address}" + + echo -e "\n${YELLOW}System Information:${NC}" + echo -e "Internal IP: ${host}" + echo -e "System Arch: $(uname -m) (identified as: ${arch})" + echo -e "Plugins Dir: ${PLUGINS_DIR}" + echo -e "Chart Dir: ${DEPLOY_DIR}/chart/" + + echo -e "${BLUE}Operation Guide:${NC}" + echo -e "1. Check cluster status: kubectl get pod -n $NAMESPACE" + echo -e "2. View real-time logs: kubectl logs -f -n $NAMESPACE " + echo -e " Example: kubectl logs -f framework-deploy-5577c87b6-h82g8 -n euler-copilot" + echo -e "3. Check POD status: kubectl get pods -n $NAMESPACE" + + +} + +main "$@" diff --git a/deploy/en/scripts/9-other-script/backup_databases.sh b/deploy/en/scripts/9-other-script/backup_databases.sh new file mode 100755 index 000000000..1de4f101a --- /dev/null +++ b/deploy/en/scripts/9-other-script/backup_databases.sh @@ -0,0 +1,289 @@ +#!/bin/bash +set -e + +# Color definitions +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +PURPLE='\033[0;35m' +CYAN='\033[0;36m' +NC='\033[0m' # No Color + +# Backup directories +BACKUP_BASE="/home/dump" +MYSQL_BACKUP_DIR="$BACKUP_BASE/mysql" +OPENGAUSS_BACKUP_DIR="$BACKUP_BASE/opengauss" +MINIO_BACKUP_DIR="$BACKUP_BASE/minio" +OPENGAUSS_DATA_PATH="/home/omm/opengauss.sql" +MYSQL_DATA_PATH="/home/mysql.sql" +MYSQL_USER="authhub" +MYSQL_DB_NAME="oauth2" +MYSQL_TABLE_NAME="user" +GS_USERNAME="postgres" +GS_DB="postgres" + +# Timestamp function +timestamp() { + echo -n "$(date '+%Y-%m-%d %H:%M:%S')" +} + +# Log functions +log_info() { + echo -e "$(timestamp) ${BLUE}=== $1 ${NC}" +} + +log_success() { + echo -e "$(timestamp) ${GREEN}$1${NC}" +} + +log_warning() { + echo -e "$(timestamp) ${YELLOW}$1${NC}" +} + +log_error() { + echo -e "$(timestamp) ${RED}$1${NC}" +} + +log_step() { + echo -e "$(timestamp) ${PURPLE} $1 ${NC}" +} + +# Print banner +print_banner() { + echo -e "${GREEN}" + echo "================================================================" + echo " Euler Copilot Data Backup Script" + echo " MySQL + OpenGauss + MinIO" + echo "================================================================" + echo -e "${NC}" +} + +print_completion_banner() { + echo -e "${GREEN}" + echo "================================================================" + echo " MySQL + OpenGauss + MinIO Data Backup Completed!" + echo "================================================================" + echo -e "${NC}" +} + +# Check command execution result +check_command() { + if [ $? -eq 0 ]; then + log_success "$1" + else + log_error "$2" + exit 1 + fi +} + +# Create backup directories +create_backup_directories() { + log_step "Creating backup directories" + + mkdir -p "$MYSQL_BACKUP_DIR" "$OPENGAUSS_BACKUP_DIR" "$MINIO_BACKUP_DIR" + check_command "Backup directories created" "Failed to create backup directories" + + log_info "MySQL backup directory: $MYSQL_BACKUP_DIR" + log_info "OpenGauss backup directory: $OPENGAUSS_BACKUP_DIR" + log_info "MinIO backup directory: $MINIO_BACKUP_DIR" +} + +# Backup MySQL data +backup_mysql() { + log_step "Starting MySQL data backup" + + # Get MySQL Pod name + log_info "Finding MySQL Pod..." + local pod_name=$(kubectl get pod -n euler-copilot | grep mysql | awk '{print $1}') + if [ -z "$pod_name" ]; then + log_error "MySQL Pod not found" + return 1 + fi + log_success "Found Pod: $pod_name" + + # Get MySQL password + log_info "Getting MySQL password..." + local mysql_password=$(kubectl get secret authhub-secret -n euler-copilot -o jsonpath='{.data.mysql-password}' | base64 --decode) + if [ -z "$mysql_password" ]; then + log_error "Failed to get MySQL password" + return 1 + fi + log_success "Password obtained successfully" + + # Export user table + log_info "Exporting user table data..." + kubectl exec $pod_name -n euler-copilot -- bash -c "mysqldump -u${MYSQL_USER} -p${mysql_password} --no-tablespaces ${MYSQL_DB_NAME} ${MYSQL_TABLE_NAME} > ${MYSQL_DATA_PATH}" + check_command "User table export completed" "User table export failed" + + # Copy backup file to local + log_info "Copying backup file to local..." + kubectl cp $pod_name:${MYSQL_DATA_PATH} $MYSQL_BACKUP_DIR/mysql.sql -n euler-copilot + check_command "MySQL backup file copy completed" "MySQL backup file copy failed" + + # Verify backup file + if [ -f "$MYSQL_BACKUP_DIR/mysql.sql" ]; then + local file_size=$(du -h "$MYSQL_BACKUP_DIR/mysql.sql" | cut -f1) + log_success "MySQL backup completed, file size: $file_size" + else + log_error "MySQL backup file not found" + return 1 + fi +} + +# Backup OpenGauss data +backup_opengauss() { + log_step "Starting OpenGauss data backup" + + # Get OpenGauss Pod name + log_info "Finding OpenGauss Pod..." + local pod_name=$(kubectl get pod -n euler-copilot | grep opengauss | awk '{print $1}') + if [ -z "$pod_name" ]; then + log_error "OpenGauss Pod not found" + return 1 + fi + log_success "Found Pod: $pod_name" + + # Get OpenGauss password + log_info "Getting OpenGauss password..." + local gauss_password=$(kubectl get secret euler-copilot-database -n euler-copilot -o jsonpath='{.data.gauss-password}' | base64 --decode) + if [ -z "$gauss_password" ]; then + log_error "Failed to get OpenGauss password" + return 1 + fi + log_success "Password obtained successfully" + + # Export database + log_info "Exporting OpenGauss database..." + kubectl exec -it "$pod_name" -n euler-copilot -- /bin/sh -c "su - omm -c 'source ~/.bashrc && gs_dump -U ${GS_USERNAME} -f ${OPENGAUSS_DATA_PATH} -p 5432 ${GS_DB} -F p -W \"${gauss_password}\"'" + check_command "OpenGauss database export completed" "OpenGauss database export failed" + + # Copy backup file to local + log_info "Copying backup file to local..." + kubectl cp $pod_name:${OPENGAUSS_DATA_PATH} $OPENGAUSS_BACKUP_DIR/opengauss.sql -n euler-copilot + check_command "OpenGauss backup file copy completed" "OpenGauss backup file copy failed" + + # Verify backup file + if [ -f "$OPENGAUSS_BACKUP_DIR/opengauss.sql" ]; then + local file_size=$(du -h "$OPENGAUSS_BACKUP_DIR/opengauss.sql" | cut -f1) + log_success "OpenGauss backup completed, file size: $file_size" + else + log_error "OpenGauss backup file not found" + return 1 + fi +} + +# Backup MinIO data +backup_minio() { + log_step "Starting MinIO data backup" + + # Get PV name + log_info "Finding MinIO PV..." + local pv_name=$(kubectl get pv -n euler-copilot | grep minio | awk '{print $1}') + if [ -z "$pv_name" ]; then + log_error "MinIO PV not found" + return 1 + fi + log_success "Found PV: $pv_name" + + # MinIO data directory + local minio_storage_dir="/var/lib/rancher/k3s/storage/${pv_name}_euler-copilot_minio-storage/" + + # Check if source directory exists + if [ ! -d "$minio_storage_dir" ]; then + log_error "MinIO storage directory does not exist: $minio_storage_dir" + return 1 + fi + + # Backup MinIO data + log_info "Starting MinIO data backup..." + cp -r "$minio_storage_dir"* "$MINIO_BACKUP_DIR"/ + check_command "MinIO data backup completed" "MinIO data backup failed" + + # Verify backup + local source_count=$(find "$minio_storage_dir" -type f 2>/dev/null | wc -l) + local backup_count=$(find "$MINIO_BACKUP_DIR" -type f 2>/dev/null | wc -l) + + log_info "Source file count: $source_count, Backup file count: $backup_count" + + if [ "$source_count" -eq "$backup_count" ]; then + log_success "MinIO backup verification successful" + else + log_warning "File count mismatch, but backup completed" + fi +} + +# Show backup summary +show_backup_summary() { + log_step "Backup Summary" + + echo -e "${CYAN}" + echo "Backup Location: $BACKUP_BASE" + echo "----------------------------------------" + + # MySQL backup information + if [ -f "$MYSQL_BACKUP_DIR/mysql.sql" ]; then + local mysql_size=$(du -h "$MYSQL_BACKUP_DIR/mysql.sql" | cut -f1) + echo "MySQL: $MYSQL_BACKUP_DIR/mysql.sql ($mysql_size)" + else + echo "MySQL: Backup failed" + fi + + # OpenGauss backup information + if [ -f "$OPENGAUSS_BACKUP_DIR/opengauss.sql" ]; then + local gauss_size=$(du -h "$OPENGAUSS_BACKUP_DIR/opengauss.sql" | cut -f1) + echo "OpenGauss: $OPENGAUSS_BACKUP_DIR/opengauss.sql ($gauss_size)" + else + echo "OpenGauss: Backup failed" + fi + + # MinIO backup information + if [ -d "$MINIO_BACKUP_DIR" ]; then + local minio_size=$(du -sh "$MINIO_BACKUP_DIR" 2>/dev/null | cut -f1 || echo "Unknown") + local minio_count=$(find "$MINIO_BACKUP_DIR" -type f 2>/dev/null | wc -l) + echo "MinIO: $MINIO_BACKUP_DIR/ ($minio_size, $minio_count files)" + else + echo "MinIO: Backup failed" + fi + echo -e "${NC}" +} + +# Main function +main() { + print_banner + + log_step "Starting data backup process" + + # Create backup directories + create_backup_directories + + # Backup MySQL + if backup_mysql; then + log_success "MySQL backup successful" + else + log_error "MySQL backup failed" + fi + + # Backup OpenGauss + if backup_opengauss; then + log_success "OpenGauss backup successful" + else + log_error "OpenGauss backup failed" + fi + + # Backup MinIO + if backup_minio; then + log_success "MinIO backup successful" + else + log_error "MinIO backup failed" + fi + + # Show backup summary + show_backup_summary + + print_completion_banner + log_success "Data backup process completed" +} + +# Execute main function +main "$@" diff --git a/deploy/en/scripts/9-other-script/download_file.sh b/deploy/en/scripts/9-other-script/download_file.sh new file mode 100755 index 000000000..8bd11575e --- /dev/null +++ b/deploy/en/scripts/9-other-script/download_file.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +download_and_extract_files() { + local BASE_URL=$1 + local TARGET_DIR="/home/EulerCopilot/models" + shift + local FILES=("$@") + + yum -y install tar wget + + if [ ! -d "${TARGET_DIR}" ]; then + echo "Creating directory ${TARGET_DIR}..." + mkdir -p "${TARGET_DIR}" + fi + + for FILE in "${FILES[@]}"; do + FULL_URL="${BASE_URL}${FILE}" + + if [ ! -f "${FILE}" ]; then + echo "Downloading ${FULL_URL}..." + wget -O "${FILE}" "${FULL_URL}" + if [ $? -ne 0 ]; then + echo "Failed to download ${FILE}." + continue + fi + else + echo "${FILE} already exists, skipping download." + fi + + echo "Extracting ${FILE} to ${TARGET_DIR}..." + if [[ "${FILE}" == *.tar.gz ]]; then + if ! tar -xzvf "${FILE}" -C "${TARGET_DIR}" 2>&1 | grep -q 'Error is not recoverable'; then + echo "${FILE} extracted successfully." + rm "${FILE}" + else + echo "Failed to extract ${FILE}: it may be corrupt or not a tar.gz file." + rm "${FILE}" + fi + else + echo "Unsupported file format: ${FILE}" + continue + fi + done +} + +BASE_URL="https://repo.oepkgs.net/openEuler/rpm/openEuler-22.03-LTS/contrib/EulerCopilot/" +FILES=("bge-mixed-model.tar.gz" "text2vec-base-chinese-paraphrase.tar.gz" "bge-reranker-large.tar.gz") + +download_and_extract_files "${BASE_URL}" "${FILES[@]}" diff --git a/deploy/en/scripts/9-other-script/get_client_id_and_secret.py b/deploy/en/scripts/9-other-script/get_client_id_and_secret.py new file mode 100755 index 000000000..cb71e116f --- /dev/null +++ b/deploy/en/scripts/9-other-script/get_client_id_and_secret.py @@ -0,0 +1,172 @@ +""" +Get Authentication Information +""" +import json +import sys +import requests +import urllib3 +import subprocess +import argparse + +urllib3.disable_warnings() + +def get_service_cluster_ip(namespace, service_name): + cmd = ["kubectl", "get", "service", service_name, "-n", namespace, "-o", "json"] + result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + if result.returncode != 0: + error_msg = result.stderr.decode().strip() + print(f"Failed to get service info: [Namespace: {namespace}] [Service: {service_name}]") + print(f"Kubectl error details: {error_msg}") + + if "NotFound" in error_msg: + print("→ Please check:") + print(f" 1. Whether the service is deployed (kubectl get pods -n {namespace})") + print(f" 2. Whether the service name is spelled correctly") + print(f" 3. Whether you are in the correct Kubernetes context") + sys.exit(1) + + service_info = json.loads(result.stdout.decode()) + return service_info['spec'].get('clusterIP', 'No Cluster IP found') + +def get_user_token(authhub_web_url, username="administrator", password="changeme"): + url = authhub_web_url + "/oauth2/manager-login" + response = requests.post( + url, + json={"password": password, "username": username}, + headers={"Content-Type": "application/json"}, + verify=False, + timeout=10 + ) + response.raise_for_status() + return response.json()["data"]["user_token"] + +def find_existing_app(authhub_web_url, user_token, client_name): + response = requests.get( + authhub_web_url + "/oauth2/applications", + headers={"Authorization": user_token, "Content-Type": "application/json"}, + timeout=10 + ) + response.raise_for_status() + apps_data = response.json() + + for app in apps_data["data"]["applications"]: + client_metadata = app.get("client_metadata") or {} + if isinstance(client_metadata, str): + try: + client_metadata = json.loads(client_metadata) + except json.JSONDecodeError: + client_metadata = {} + + candidate_names = [ + client_metadata.get("client_name"), + app.get("client_name"), + app.get("client_info", {}).get("client_name") + ] + + if any(str(name).lower() == client_name.lower() for name in candidate_names if name): + return app["client_info"]["client_id"] + return None + +def register_or_update_app(authhub_web_url, user_token, client_name, client_url, redirect_urls): + client_id = find_existing_app(authhub_web_url, user_token, client_name) + + if client_id: + # Update existing application + print(f"Found existing application [Name: {client_name}], updating...") + url = f"{authhub_web_url}/oauth2/applications/{client_id}" + response = requests.put( + url, + json={ + "client_uri": client_url, + "redirect_uris": redirect_urls, + "register_callback_uris": [], + "logout_callback_uris": [], + "skip_authorization": True, + "scope": ["email", "phone", "username", "openid", "offline_access"], + "grant_types": ["authorization_code"], + "response_types": ["code"], + "token_endpoint_auth_method": "none" + }, + headers={"Authorization": user_token, "Content-Type": "application/json"}, + verify=False + ) + response.raise_for_status() + return response.json()["data"] + else: + # Register new application + print(f"No existing application found [Name: {client_name}], registering new application...") + response = requests.post( + authhub_web_url + "/oauth2/applications/register", + json={ + "client_name": client_name, + "client_uri": client_url, + "redirect_uris": redirect_urls, + "register_callback_uris": [], + "logout_callback_uris": [], + "skip_authorization": True, + "scope": ["email", "phone", "username", "openid", "offline_access"], + "grant_types": ["authorization_code"], + "response_types": ["code"], + "token_endpoint_auth_method": "none" + }, + headers={"Authorization": user_token, "Content-Type": "application/json"}, + verify=False + ) + response.raise_for_status() + return response.json()["data"] + +def get_client_secret(authhub_web_url, user_token, client_id): + response = requests.get( + f"{authhub_web_url}/oauth2/applications/{client_id}", + headers={"Authorization": user_token, "Content-Type": "application/json"}, + timeout=10 + ) + response.raise_for_status() + app_data = response.json() + return { + "client_id": app_data["data"]["client_info"]["client_id"], + "client_secret": app_data["data"]["client_info"]["client_secret"] + } + +if __name__ == "__main__": + # Parse command line arguments + parser = argparse.ArgumentParser() + parser.add_argument("eulercopilot_address", help="EulerCopilot frontend address (default:http://172.0.0.1:30080)") + args = parser.parse_args() + + # Get service information + namespace = "euler-copilot" + service_name = "authhub-web-service" + print(f"Querying service info: [Namespace: {namespace}] [Service: {service_name}]") + cluster_ip = get_service_cluster_ip(namespace, service_name) + authhub_web_url = f"http://{cluster_ip}:8000" + + # Generate fixed URLs + client_url = f"{args.eulercopilot_address}" + redirect_urls = [f"{args.eulercopilot_address}/api/auth/login"] + client_name = "EulerCopilot" # Set fixed default value + + # Authentication process + try: + print("\nGetting user token...") + user_token = get_user_token(authhub_web_url) + print("✓ User token obtained successfully") + + print(f"\nProcessing application [Name: {client_name}]...") + app_info = register_or_update_app(authhub_web_url, user_token, client_name, client_url, redirect_urls) + print("✓ Application processed successfully") + + print(f"\nQuerying client credentials [ID: {app_info['client_info']['client_id']}]...") + client_info = get_client_secret(authhub_web_url, user_token, app_info["client_info"]["client_id"]) + + print("\n✓ Authentication information obtained successfully:") + print(f"client_id: {client_info['client_id']}") + print(f"client_secret: {client_info['client_secret']}") + + except requests.exceptions.HTTPError as e: + print(f"\nHTTP Error: {e.response.status_code} - {e.response.text}") + sys.exit(1) + except Exception as e: + print(f"\nError: {str(e)}") + sys.exit(1) diff --git a/deploy/en/scripts/9-other-script/get_log.sh b/deploy/en/scripts/9-other-script/get_log.sh new file mode 100755 index 000000000..32eef6451 --- /dev/null +++ b/deploy/en/scripts/9-other-script/get_log.sh @@ -0,0 +1,37 @@ +#!/bin/bash +function help { + echo -e "Usage: ./get_log.sh [namespace] [log_time]"; + echo -e "Example: ./get_log.sh euler-copilot 1h"; +} + +function main { + echo -e "[Info] Starting to collect pod logs"; + time=$(date -u +"%s"); + echo -e "[Info] Current namespace: $1, Current timestamp: $time" + filename="logs_$1_$time"; + + mkdir $filename; + echo $time > $filename/timestamp; + + echo "[Info] Starting log collection"; + kubectl -n $1 events > $filename/events.log; + + pod_names=$(kubectl -n $1 get pods -o name); + while IFS= read -r line || [[ -n $line ]]; do + mkdir -p $filename/$line; + kubectl -n $1 describe $line > $filename/$line/details.log; + kubectl -n $1 logs --previous --since $2 --all-containers=true --ignore-errors=true $line > $filename/$line/previous.log; + kubectl -n $1 logs --since $2 --all-containers=true --ignore-errors=true $line > $filename/$line/current.log; + done < <(printf '%s' "$pod_names"); + + tar -czf $filename.tar.gz $filename/; + rm -rf $filename; + + echo -e "[Info] Log collection completed, please provide $filename.tar.gz to us for analysis"; +} + +if [[ $# -lt 2 ]]; then + help +else + main $1 $2; +fi diff --git a/deploy/en/scripts/9-other-script/import_images.sh b/deploy/en/scripts/9-other-script/import_images.sh new file mode 100755 index 000000000..df8523380 --- /dev/null +++ b/deploy/en/scripts/9-other-script/import_images.sh @@ -0,0 +1,258 @@ +#!/bin/bash + +# Color definitions +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +MAGENTA='\033[0;35m' +CYAN='\033[0;36m' +NC='\033[0m' # Reset color + +# Default configuration +DEFAULT_VERSION="0.9.5" +IMAGE_BASE_DIR="/home/eulercopilot/images" + +# Show help information +show_help() { + echo -e "${YELLOW}Usage:${NC}" + echo -e " $0 [options] [parameters]" + echo -e "${YELLOW}Options:${NC}" + echo -e " -v, --version \tSpecify image version (default: ${DEFAULT_VERSION})" + echo -e " -i, --import \t\tImport single image file" + echo -e " -d, --delete \t\tDelete specified image (format: repo/image:tag)" + echo -e " --delete-all\t\t\tDelete all neocopilot images" + echo -e " -h, --help\t\t\tShow help information" + echo -e "${YELLOW}Examples:${NC}" + echo -e " $0 -v 0.9.4\t\t\tImport version 0.9.4 images" + echo -e " $0 -i $IMAGE_BASE_DIR/$DEFAULT_VERSION/myimage.tar\tImport single image file" + echo -e " $0 -d hub.oepkgs.net/neocopilot/authhub:0.9.3-arm\tDelete specified image" + echo -e " $0 --delete-all\t\tDelete all neocopilot images" + exit 0 +} + +# Parameter parsing +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + -v|--version) + eulercopilot_version="$2" + shift 2 + ;; + -i|--import) + single_image_file="$2" + shift 2 + ;; + -d|--delete) + image_to_delete="$2" + shift 2 + ;; + --delete-all) + delete_all_images=true + shift + ;; + -h|--help) + show_help + ;; + *) + echo -e "${RED}Unknown parameter: $1${NC}" + show_help + exit 1 + ;; + esac + done +} + +# System architecture detection +detect_architecture() { + ARCH=$(uname -m) + case $ARCH in + x86_64) + ARCH_SUFFIX="x86" + ;; + aarch64|armv*) + ARCH_SUFFIX="arm" + ;; + *) + echo -e "${RED}Unsupported architecture: $ARCH${NC}" + exit 1 + ;; + esac +} + +# Delete specified image +delete_image() { + local image=$1 + echo -e "${YELLOW}Deleting image: $image${NC}" + if sudo k3s ctr -n=k8s.io images rm "$image"; then + echo -e "${GREEN} Deletion successful${NC}" + else + echo -e "${RED} Deletion failed${NC}" + fi +} + +# Delete all neocopilot images +delete_all_neocopilot_images() { + echo -e "${YELLOW}Deleting all neocopilot images...${NC}" + sudo k3s crictl images | grep neocopilot | awk '{print $1":"$2}' | while read -r image; do + delete_image "$image" + done +} + +# Import single image file +import_single_image() { + local file=$1 + echo -e "${CYAN}Importing single image: $file${NC}" + + if [[ ! -f "$file" ]]; then + echo -e "${RED}Error: File does not exist $file${NC}" + exit 1 + fi + + if sudo k3s ctr -n=k8s.io images import "$file"; then + echo -e "${GREEN} Import successful${NC}" + else + echo -e "${RED} Import failed${NC}" + exit 1 + fi +} + +# Batch import images +batch_import_images() { + local version=$1 + local image_dir="$IMAGE_BASE_DIR/$version" + + if [[ ! -d "$image_dir" ]]; then + echo -e "${RED}Error: Image directory does not exist $image_dir${NC}" + exit 1 + fi + + echo -e "${CYAN}Scanning directory: $image_dir${NC}" + echo -e "${CYAN}Found the following TAR files:${NC}" + ls -1 "$image_dir"/*.tar + + local success_count=0 + local fail_count=0 + + for tar_file in "$image_dir"/*.tar; do + [[ -f "$tar_file" ]] || continue + + echo -e "\n${BLUE}Importing $tar_file...${NC}" + + if sudo k3s ctr -n=k8s.io images import "$tar_file"; then + ((success_count++)) + echo -e "${GREEN} Import successful${NC}" + else + ((fail_count++)) + echo -e "${RED} Import failed${NC}" + fi + done + + echo -e "\n${CYAN}Import results:${NC}" + echo -e "${GREEN}Successful: $success_count${NC}" + echo -e "${RED}Failed: $fail_count${NC}" + + return $fail_count +} + +# Image integrity check +check_images() { + local version=$1 + local missing_count=0 + + # Base image list (using version variable) + local base_images=( + "hub.oepkgs.net/neocopilot/euler-copilot-framework:${version}-arm" + "hub.oepkgs.net/neocopilot/euler-copilot-web:${version}-arm" + "hub.oepkgs.net/neocopilot/data_chain_back_end:${version}-arm" + "hub.oepkgs.net/neocopilot/data_chain_web:${version}-arm" + "hub.oepkgs.net/neocopilot/authhub:0.9.3-arm" + "hub.oepkgs.net/neocopilot/authhub-web:0.9.3-arm" + "hub.oepkgs.net/neocopilot/opengauss:latest-arm" + "hub.oepkgs.net/neocopilot/redis:7.4-alpine-arm" + "hub.oepkgs.net/neocopilot/mysql:8-arm" + "hub.oepkgs.net/neocopilot/minio:empty-arm" + "hub.oepkgs.net/neocopilot/mongo:7.0.16-arm" + "hub.oepkgs.net/neocopilot/secret_inject:dev-arm" + ) + + # Adjust expected image tags based on architecture + local expected_images=() + for image in "${base_images[@]}"; do + if [[ "$ARCH_SUFFIX" == "x86" ]]; then + expected_image="${image/-arm/-x86}" + else + expected_image="$image" + fi + expected_images+=("$expected_image") + done + + echo -e "\n${MAGENTA}Starting image integrity check:${NC}" + for image in "${expected_images[@]}"; do + if sudo k3s ctr -n=k8s.io images ls | grep -q "$image"; then + echo -e "${GREEN}[Exists] $image${NC}" + else + echo -e "${RED}[Missing] $image${NC}" + ((missing_count++)) + fi + done + + echo -e "\n${MAGENTA}Checking images with crictl:${NC}" + sudo k3s crictl images | grep neocopilot + + if [[ $missing_count -gt 0 ]]; then + echo -e "\n${RED}Warning: Missing $missing_count required images${NC}" + return 1 + else + echo -e "\n${GREEN}All required images are ready${NC}" + return 0 + fi +} + +# Main function +main() { + parse_args "$@" + detect_architecture + + # Handle delete operations + if [[ -n "$image_to_delete" ]]; then + delete_image "$image_to_delete" + exit 0 + fi + + if [[ "$delete_all_images" == true ]]; then + delete_all_neocopilot_images + exit 0 + fi + + # Handle single image import + if [[ -n "$single_image_file" ]]; then + import_single_image "$single_image_file" + exit 0 + fi + + # Default batch import mode + local version=${eulercopilot_version:-$DEFAULT_VERSION} + + echo -e "${YELLOW}==============================${NC}" + echo -e "${CYAN}Architecture\t: ${ARCH_SUFFIX}${NC}" + echo -e "${CYAN}Target version\t: ${version}${NC}" + echo -e "${CYAN}Image directory\t: ${IMAGE_BASE_DIR}/${version}${NC}" + echo -e "${YELLOW}==============================${NC}" + + batch_import_images "$version" + import_result=$? + + if [[ $import_result -eq 0 ]]; then + check_images "$version" || exit 1 + else + echo -e "${RED}Some images failed to import, skipping integrity check${NC}" + exit 1 + fi + + echo -e "${GREEN}System ready, all images available${NC}" +} + +# Execute main function +main "$@" +exit 0 diff --git a/deploy/en/scripts/9-other-script/install_oidc_eulercopilot.sh b/deploy/en/scripts/9-other-script/install_oidc_eulercopilot.sh new file mode 100755 index 000000000..f098aedb8 --- /dev/null +++ b/deploy/en/scripts/9-other-script/install_oidc_eulercopilot.sh @@ -0,0 +1,268 @@ +#!/bin/bash + +# 颜色定义 +RED='\e[31m' +GREEN='\e[32m' +YELLOW='\e[33m' +BLUE='\e[34m' +NC='\e[0m' # 恢复默认颜色 + +set -eo pipefail + +# 颜色定义 +RED='\e[31m' +GREEN='\e[32m' +YELLOW='\e[33m' +BLUE='\e[34m' +NC='\e[0m' # 恢复默认颜色 + +PLUGINS_DIR="/home/eulercopilot/semantics" +SCRIPT_PATH="$( + cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 + pwd +)/$(basename "${BASH_SOURCE[0]}")" + +DEPLOY_DIR="$( + canonical_path=$(readlink -f "$SCRIPT_PATH" 2>/dev/null || echo "$SCRIPT_PATH") + dirname "$(dirname "$(dirname "$canonical_path")")" +)" + + +get_eth0_ip() { + echo -e "${BLUE}获取 eth0 网络接口 IP 地址...${NC}" + local timeout=20 + local start_time=$(date +%s) + local interface="eth0" + + # 检查 eth0 是否存在，并等待其变为可用状态 + while [ $(( $(date +%s) - start_time )) -lt $timeout ]; do + if ip link show "$interface" > /dev/null 2>&1; then + break + else + sleep 1 + fi + done + + if ! ip link show "$interface" > /dev/null 2>&1; then + echo -e "${RED}错误：未找到网络接口 ${interface}${NC}" + exit 1 + fi + + # 获取 IP 地址 + host=$(ip addr show "$interface" | grep -w inet | awk '{print $2}' | cut -d'/' -f1) + + if [[ -z "$host" ]]; then + echo -e "${RED}错误：未能从接口 ${interface} 获取 IP 地址${NC}" + exit 1 + fi + + echo -e "${GREEN}使用网络接口：${interface}，IP 地址：${host}${NC}" +} + +get_user_input() { + echo -e "${BLUE}请输入 OAuth 客户端配置：${NC}" + read -p "Client ID: " client_id + read -s -p "Client Secret: " client_secret + + echo + + # 检查必填字段 + if [[ -z "$client_id" || -z "$client_secret" ]]; then + echo -e "${RED}错误：Client Secret 存在空行，请重新输入${NC}" + read -s -p "Client Secret: " client_secret + echo + echo -e "${GREEN}Client Secret 已正确输入${NC}" + fi + + # 处理Copilot域名 + echo -e "${BLUE}请输入 EulerCopilot 域名（直接回车使用默认值 www.eulercopilot.local）：${NC}" + read -p "EulerCopilot 的前端域名: " eulercopilot_domain + + if [[ -z "$eulercopilot_domain" ]]; then + eulercopilot_domain="www.eulercopilot.local" + echo -e "${GREEN}使用默认域名：${eulercopilot_domain}${NC}" + else + if ! [[ "${eulercopilot_domain}" =~ ^([a-zA-Z0-9-]+\.)*[a-zA-Z0-9-]+\.[a-zA-Z]{2,}$ ]]; then + echo -e "${RED}错误：输入的EulerCopilot域名格式不正确${NC}" + exit 1 + fi + echo -e "${GREEN}输入域名：${eulercopilot_domain}${NC}" + fi + + # 处理Authhub域名 + echo -e "${BLUE}请输入 Authhub 的域名配置（直接回车使用默认值 authhub.eulercopilot.local）：${NC}" + read -p "Authhub 的前端域名: " authhub_domain + if [[ -z "$authhub_domain" ]]; then + authhub_domain="authhub.eulercopilot.local" + echo -e "${GREEN}使用默认域名：${authhub_domain}${NC}" + else + if ! [[ "${authhub_domain}" =~ ^([a-zA-Z0-9-]+\.)*[a-zA-Z0-9-]+\.[a-zA-Z]{2,}$ ]]; then + echo -e "${RED}错误：输入的AuthHub域名格式不正确${NC}" + exit 1 + fi + echo -e "${GREEN}输入域名：${authhub_domain}${NC}" + fi + +} + +# 检查必要目录 +check_directories() { + echo -e "${BLUE}检查语义接口目录是否存在...${NC}" + if [ -d "${PLUGINS_DIR}" ]; then + echo -e "${GREEN}目录已存在：${PLUGINS_DIR}${NC}" + else + if mkdir -p "${PLUGINS_DIR}"; then + echo -e "${GREEN}目录已创建：${PLUGINS_DIR}${NC}" + else + echo -e "${RED}错误：无法创建目录 ${PLUGINS_DIR}${NC}" + exit 1 + fi + fi +} + +# 安装前检查并删除已有部署 +check_and_delete_existing_deployment() { + echo -e "${YELLOW}检查是否存在已部署的euler-copilot...${NC}" + if helm list -n euler-copilot --short | grep -q "^euler-copilot$"; then + echo -e "${YELLOW}发现已存在的euler-copilot部署，正在删除...${NC}" + helm uninstall -n euler-copilot euler-copilot + + if [ $? -ne 0 ]; then + echo -e "${RED}错误：删除旧版euler-copilot失败${NC}" + exit 1 + fi + + echo -e "${YELLOW}等待旧部署清理完成（10秒）...${NC}" + sleep 10 + else + echo -e "${GREEN}未找到已存在的euler-copilot部署，继续安装...${NC}" + fi +} + +# 修改YAML配置文件的方法 +modify_yaml() { + echo -e "${BLUE}开始修改YAML配置文件...${NC}" + cd + python3 ${SCRIPTS_DIR}/8-install-EulerCopilot/modify_eulercopilot_yaml.py \ + "${CHART_DIR}/euler_copilot/values.yaml" \ + "${CHART_DIR}/euler_copilot/values.yaml" \ + --set "models.answer.url=http://120.46.78.178:8000" \ + --set "models.answer.key=sk-EulerCopilot1bT1WtG2ssG92pvOPTkpT3BlbkFJVruTv8oUe" \ + --set "models.answer.name=Qwen2.5-32B-Instruct-GPTQ-Int4" \ + --set "models.answer.ctx_length=8192" \ + --set "models.answer.max_tokens=2048" \ + --set "models.embedding.url=https://192.168.50.4:8001/embedding/v1" \ + --set "models.embedding.key=sk-123456" \ + --set "models.embedding.name=bge-m3" \ + --set "login.type=oidc" \ + --set "login.client.id=623c3c2f1eca5ad5fca6c58a" \ + --set "login.client.secret=5d07c65f44fa1beb08b36f90af314aef" \ + --set "login.oidc.token_url=https://omapi.test.osinfra.cn/oneid/oidc/token" \ + --set "login.oidc.user_url=https://omapi.test.osinfra.cn/oneid/oidc/user" \ + --set "login.oidc.redirect=https://omapi.test.osinfra.cn/oneid/oidc/authorize?client_id=623c3c2f1eca5ad5fca6c58a&redirect_uri=https://qa-robot-openeuler.test.osinfra.cn/api/auth/login&scope=openid+profile+email+phone+offline_access&complementation=phone&access_type=offline&response_type=code" \ + --set "domain.euler_copilot=qa-robot-eulercopilot.test.osinfra.cn" \ + + if [ $? -ne 0 ]; then + echo -e "${RED}错误：YAML文件修改失败${NC}" + exit 1 + fi + echo -e "${GREEN}YAML文件修改成功！${NC}" +} + +# 进入Chart目录的方法 +enter_chart_directory() { + echo -e "${BLUE}进入Chart目录...${NC}" + cd "${DEPLOY_DIR}/chart/" || { + echo -e "${RED}错误：无法进入Chart目录 ${DEPLOY_DIR}/chart/${NC}" + exit 1 + } +} + +# 执行Helm安装的方法 +execute_helm_install() { + echo -e "${BLUE}开始部署EulerCopilot...${NC}" + helm install -n euler-copilot euler-copilot ./euler_copilot + + if [ $? -ne 0 ]; then + echo -e "${RED}错误：Helm安装失败${NC}" + exit 1 + fi + echo -e "${GREEN}Helm安装EulerCopilot成功！${NC}" +} + +check_pods_status() { + echo -e "${BLUE}==> 等待初始化就绪（30秒）...${NC}" + sleep 30 + + local timeout=100 + local start_time=$(date +%s) + + echo -e "${BLUE}开始监控Pod状态（总超时时间300秒）...${NC}" + echo -e "${BLUE}镜像拉取中...${NC}" + + while true; do + local current_time=$(date +%s) + local elapsed=$((current_time - start_time)) + + # 超时处理逻辑 + if [ $elapsed -gt $timeout ]; then + echo -e "${YELLOW}警告：部署超时！请检查以下Pod状态：${NC}" + kubectl get pods -n euler-copilot + echo -e "${YELLOW}注意：部分Pod可能仍在启动中，可稍后手动检查${NC}" + return 1 + fi + + # 检查所有Pod状态 + local not_running=$( + kubectl get pods -n euler-copilot -o jsonpath='{range .items[*]}{.metadata.name} {.status.phase} {.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' \ + | awk '$2 != "Running" || $3 != "True" {print $1 " " $2}' + ) + + if [ -z "$not_running" ]; then + echo -e "${GREEN}所有Pod已正常运行！${NC}" + kubectl get pods -n euler-copilot + return 0 + else + echo "等待Pod就绪（已等待 ${elapsed} 秒）..." + echo "当前未启动Pod：" + echo "$not_running" | awk '{print " - " $1 " (" $2 ")"}' + sleep 10 + fi + done +} + +# 主函数执行各个步骤 +main() { + get_eth0_ip + get_user_input + check_directories + check_and_delete_existing_deployment + modify_yaml + enter_chart_directory + execute_helm_install + + # Pod状态检查并处理结果 + if check_pods_status; then + echo -e "${GREEN}所有组件已就绪！${NC}" + else + echo -e "${YELLOW}注意：部分组件尚未就绪，可稍后手动检查${NC}" + fi + + # 最终部署信息输出 + echo -e "\n${GREEN}==================================================${NC}" + echo -e "${GREEN} EulerCopilot 部署完成！ ${NC}" + echo -e "${GREEN}==================================================${NC}" + echo -e "${YELLOW}EulerCopilot访问地址：\thttps://${eulercopilot_domain}${NC}" + echo -e "${YELLOW}AuthHub管理地址：\thttps://${authhub_domain}${NC}" + echo -e "${YELLOW}插件目录：\t\t${PLUGINS_DIR}${NC}" + echo -e "${YELLOW}Chart目录：\t${DEPLOY_DIR}/chart/${NC}" + echo + echo -e "${BLUE}温馨提示：" + echo -e "${BLUE}1. 请确保域名已正确解析到集群Ingress地址${NC}" + echo -e "${BLUE}2. 首次拉取RAG镜像可能需要约1-3分钟,POD会稍后自动启动${NC}" + echo -e "${BLUE}3. 查看实时状态：kubectl get pods -n euler-copilot${NC}" + echo -e "${BLUE}4. 查看镜像：k3s crictl images${NC}" +} + +# 调用主函数 +main diff --git a/deploy/en/scripts/9-other-script/migrate_minio_database.sh b/deploy/en/scripts/9-other-script/migrate_minio_database.sh new file mode 100755 index 000000000..bc17e8b96 --- /dev/null +++ b/deploy/en/scripts/9-other-script/migrate_minio_database.sh @@ -0,0 +1,244 @@ +#!/bin/bash +set -e + +# Color definitions +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +PURPLE='\033[0;35m' +CYAN='\033[0;36m' +NC='\033[0m' # No Color + +# Log functions +log_info() { echo -e "${BLUE}[INFO]${NC} $1"; } +log_success() { echo -e "${GREEN}[SUCCESS]${NC} $1"; } +log_warning() { echo -e "${YELLOW}[WARNING]${NC} $1"; } +log_error() { echo -e "${RED}[ERROR]${NC} $1"; } +log_step() { echo -e "${PURPLE}[STEP]${NC} $1"; } + +# Global variables +PV_NAME="" +POD_NAME="" +STORAGE_DIR="" +NEW_POD_NAME="" +MINIO_PASSWORD="" +MINIO_ROOT_USER="minioadmin" +MINIO_HOST="" +MINIO_PORT="" +source_dir="/home/dump/minio" + +# Print banner +print_banner() { + echo -e "${GREEN}" + echo "====================================================================" + echo " MINIO Data Import Script" + echo " Euler Copilot Project Specific" + echo "====================================================================" + echo -e "${NC}" +} + +print_completion_banner() { + echo -e "${GREEN}" + echo "====================================================================" + echo " MINIO Data Import Completed!" + echo "====================================================================" + echo -e "${NC}" +} + +# User confirmation +confirm_execution() { + echo -e "${YELLOW}Warning: This operation will clear existing MinIO data and import new data!${NC}" + read -p "Confirm execution of MinIO data import operation? (y/N): " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + log_info "Operation cancelled" + exit 0 + fi +} + +# Remove color codes and special characters +clean_output() { + echo "$1" | sed -r "s/\x1B\[([0-9]{1,3}(;[0-9]{1,2})?)?[mGK]//g" | tr -d '\n' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' +} + +# Check command execution result +check_command() { + if [ $? -eq 0 ]; then + log_success "$1" + else + log_error "$2" + exit 1 + fi +} + +# Check prerequisites +check_prerequisites() { + log_step "Checking prerequisites..." + + # Check if kubectl is available + kubectl cluster-info &> /dev/null + check_command "kubectl connection normal" "Unable to connect to Kubernetes cluster" + + # Check if namespace exists + if ! kubectl get namespace euler-copilot &> /dev/null; then + log_error "Namespace euler-copilot does not exist" + exit 1 + fi + log_success "Namespace euler-copilot exists" +} + +# Get Kubernetes resources +get_kubernetes_resources() { + log_step "Getting Kubernetes resource information..." + + # Get PV name + PV_NAME=$(kubectl get pv -n euler-copilot | grep minio | awk '{print $1}') + if [ -z "$PV_NAME" ]; then + log_error "No MinIO PV found" + exit 1 + fi + log_info "PV name: $PV_NAME" + + # Get Pod name + POD_NAME=$(kubectl get pods -n euler-copilot | grep minio | grep Running | awk '{print $1}') + POD_NAME=$(clean_output "$POD_NAME") + if [ -z "$POD_NAME" ]; then + log_error "No running MinIO Pod found" + exit 1 + fi + log_info "Pod name: $POD_NAME" + + # Set storage directory + STORAGE_DIR="/var/lib/rancher/k3s/storage/${PV_NAME}_euler-copilot_minio-storage/" + log_info "Storage directory: $STORAGE_DIR" +} + +# Copy data +copy_data() { + log_step "Copying data..." + + if [ -d "$source_dir" ]; then + # Check source data + local source_count=$(find "$source_dir" -type f 2>/dev/null | wc -l) + if [ "$source_count" -eq 0 ]; then + log_warning "Source directory is empty, skipping copy" + return 0 + fi + + log_info "Source directory file count: $source_count" + cp -r "$source_dir"/* "$STORAGE_DIR" + check_command "Data copy completed" "Data copy failed" + + # Verify copy result + local new_count=$(find "$STORAGE_DIR" -type f 2>/dev/null | wc -l) + log_info "File count after copy: $new_count" + + if [ "$source_count" -eq "$new_count" ]; then + log_success "File count verification successful" + else + log_warning "File count mismatch: source=$source_count, target=$new_count" + fi + else + log_error "Source directory $source_dir does not exist" + exit 1 + fi +} + +# Restart Pod +restart_pod() { + log_step "Restarting Pod..." + + kubectl delete pod "$POD_NAME" -n euler-copilot + check_command "Pod deletion command executed successfully" "Pod deletion failed" + + # Wait for Pod restart + log_info "Waiting for Pod restart..." + local timeout=60 + local counter=0 + + while [ $counter -lt $timeout ]; do + NEW_POD_NAME=$(kubectl get pods -n euler-copilot | grep minio | grep Running | awk '{print $1}') + NEW_POD_NAME=$(clean_output "$NEW_POD_NAME") + + if [ -n "$NEW_POD_NAME" ]; then + log_success "Pod restarted successfully: $NEW_POD_NAME" + return 0 + fi + + counter=$((counter + 5)) + sleep 5 + echo -n "." + done + + log_error "Pod did not restart within specified time" + return 1 +} + +# Get MinIO configuration +get_minio_config() { + log_step "Getting MinIO configuration..." + + MINIO_PASSWORD=$(kubectl get secret euler-copilot-database -n euler-copilot -o jsonpath='{.data.minio-password}' 2>/dev/null | base64 --decode) + if [ $? -ne 0 ] || [ -z "$MINIO_PASSWORD" ]; then + log_error "Failed to get MinIO password" + exit 1 + fi + + MINIO_HOST=$(kubectl get svc -n euler-copilot | grep minio-service | awk '{print $3}') + MINIO_PORT=$(kubectl get svc -n euler-copilot | grep minio-service | awk '{split($5, a, "/"); print a[1]}') + + log_info "MinIO configuration:" + log_info " Host: $MINIO_HOST" + log_info " Port: $MINIO_PORT" + log_info " User: $MINIO_ROOT_USER" +} + +# Verify MinIO data +verify_minio_data() { + log_step "Verifying MinIO data..." + + # Set up mc client + kubectl exec -it "$NEW_POD_NAME" -n euler-copilot -- mc alias set myminio "http://${MINIO_HOST}:${MINIO_PORT}" "$MINIO_ROOT_USER" "$MINIO_PASSWORD" + check_command "MinIO client setup successful" "MinIO client setup failed" + + # List buckets + log_info "MinIO Buckets list:" + kubectl exec -it "$NEW_POD_NAME" -n euler-copilot -- mc ls myminio + if [ $? -eq 0 ]; then + log_success "MinIO connection normal" + else + log_error "MinIO connection failed" + exit 1 + fi + + # Check specific bucket content + log_info "Checking witchaind-doc bucket:" + kubectl exec -it "$NEW_POD_NAME" -n euler-copilot -- mc ls myminio/witchaind-doc + if [ $? -eq 0 ]; then + log_success "witchaind-doc bucket access successful" + else + log_warning "witchaind-doc bucket does not exist or is empty" + fi +} + +# Main function +main() { + print_banner + confirm_execution + + # Execute each step + check_prerequisites + get_kubernetes_resources + copy_data + restart_pod + get_minio_config + verify_minio_data + + print_completion_banner + log_success "MINIO data import process fully completed" + log_info "Please verify if business functions are working normally" +} + +# Execute main function +main "$@" diff --git a/deploy/en/scripts/9-other-script/migrate_mysql_database.sh b/deploy/en/scripts/9-other-script/migrate_mysql_database.sh new file mode 100755 index 000000000..36720984d --- /dev/null +++ b/deploy/en/scripts/9-other-script/migrate_mysql_database.sh @@ -0,0 +1,262 @@ +#!/bin/bash + +# MySQL Database Recovery Script +# Description: Used for Euler Copilot project MySQL database recovery + +set -e # Exit immediately on error + +# Color definitions +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +PURPLE='\033[0;35m' +CYAN='\033[0;36m' +NC='\033[0m' # No Color + +# Print functions +print_color() { + local color=$1 + shift + echo -e "${color}$*${NC}" +} + +log() { + print_color "${GREEN}" "[$(date '+%Y-%m-%d %H:%M:%S')] $1" +} + +info() { + print_color "${BLUE}" "[$(date '+%Y-%m-%d %H:%M:%S')] $1" +} + +warning() { + print_color "${YELLOW}" "[$(date '+%Y-%m-%d %H:%M:%S')] Warning: $1" +} + +error() { + print_color "${RED}" "[$(date '+%Y-%m-%d %H:%M:%S')] Error: $1" >&2 + exit 1 +} + +step() { + echo + print_color "${PURPLE}" "[$(date '+%Y-%m-%d %H:%M:%S')] === $1 ===" +} + +# Configuration variables +NAMESPACE="euler-copilot" +SECRET_NAME="authhub-secret" +PASSWORD_KEY="mysql-password" +DB_USER="authhub" +DB_NAME="oauth2" +BACKUP_FILE="/home/dump/mysql/mysql.sql" +POD_BACKUP_PATH="/home/mysql.sql" + +# Show banner +show_banner() { + echo + print_color "${PURPLE}" "================================================" + print_color "${PURPLE}" " MySQL Database Recovery Script" + print_color "${PURPLE}" " Euler Copilot Project Specific" + print_color "${PURPLE}" "================================================" + echo +} + +# Check required tools +check_dependencies() { + step "Checking required tools" + command -v kubectl >/dev/null 2>&1 || error "kubectl not installed" + command -v base64 >/dev/null 2>&1 || error "base64 not installed" + log "Dependency check passed" +} + +# Get MySQL Pod name +get_mysql_pod() { + step "Finding MySQL Pod" + POD_NAME=$(kubectl get pod -n $NAMESPACE 2>/dev/null | grep mysql | grep Running | awk '{print $1}') + + if [ -z "$POD_NAME" ]; then + error "No running MySQL Pod found" + fi + log "Found Pod: $POD_NAME" +} + +# Get MySQL password +get_mysql_password() { + step "Getting MySQL password" + MYSQL_PASSWORD=$(kubectl get secret $SECRET_NAME -n $NAMESPACE -o jsonpath="{.data.$PASSWORD_KEY}" 2>/dev/null | base64 --decode) + + if [ -z "$MYSQL_PASSWORD" ]; then + error "Unable to get MySQL password, please check if secret exists" + fi + log "Password obtained successfully" +} + +# Check if backup file exists +check_backup_file() { + step "Checking backup file" + if [ ! -f "$BACKUP_FILE" ]; then + error "Backup file $BACKUP_FILE does not exist" + fi + + # Display file information + local file_size=$(du -h "$BACKUP_FILE" | cut -f1) + local file_lines=$(wc -l < "$BACKUP_FILE" 2>/dev/null || echo "Unknown") + info "File path: $BACKUP_FILE" + info "File size: $file_size" + info "File lines: $file_lines" + + log "Backup file check passed" +} + +# Copy backup file to Pod +copy_backup_to_pod() { + step "Copying backup file to Pod" + info "Copying from local to Pod: $BACKUP_FILE -> $POD_NAME:$POD_BACKUP_PATH" + + kubectl cp "$BACKUP_FILE" "$POD_NAME:$POD_BACKUP_PATH" -n $NAMESPACE + + if [ $? -eq 0 ]; then + log "Backup file copy completed" + else + error "File copy failed" + fi +} + +# Verify database connection +test_database_connection() { + step "Testing database connection" + info "Testing user $DB_USER connection to database $DB_NAME" + + kubectl exec $POD_NAME -n $NAMESPACE -- bash -c " + mysql -u$DB_USER -p'$MYSQL_PASSWORD' -e 'SELECT 1;' $DB_NAME 2>/dev/null + " >/dev/null 2>&1 + + if [ $? -eq 0 ]; then + log "Database connection test successful" + else + error "Database connection failed, please check password and network connection" + fi +} + +# Execute database recovery +restore_database() { + step "Executing database recovery" + warning "This operation will overwrite existing database data!" + + info "Starting database recovery..." + kubectl exec $POD_NAME -n $NAMESPACE -- bash -c " + mysql -u$DB_USER -p'$MYSQL_PASSWORD' $DB_NAME < $POD_BACKUP_PATH + " + + local restore_status=$? + + if [ $restore_status -eq 0 ]; then + log "Database recovery successful" + else + error "Database recovery failed, exit code: $restore_status" + fi +} + +# Verify recovery result +verify_restore() { + step "Verifying recovery result" + info "Checking database table information..." + + local table_count=$(kubectl exec $POD_NAME -n $NAMESPACE -- bash -c " + mysql -u$DB_USER -p'$MYSQL_PASSWORD' -N -e \\ + \"SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = '$DB_NAME';\" 2>/dev/null + " 2>/dev/null) + + if [ -n "$table_count" ] && [ "$table_count" -gt 0 ]; then + log "Recovery verification successful, database contains $table_count tables" + + # Display some table names + info "Database table list:" + kubectl exec $POD_NAME -n $NAMESPACE -- bash -c " + mysql -u$DB_USER -p'$MYSQL_PASSWORD' -e \\ + \"SELECT table_name FROM information_schema.tables WHERE table_schema = '$DB_NAME' LIMIT 10;\" 2>/dev/null + " 2>/dev/null + else + warning "Unable to get table information, but recovery operation completed" + fi +} + +# Clean up temporary files +cleanup() { + step "Cleaning up temporary files" + info "Deleting backup file in Pod: $POD_BACKUP_PATH" + + kubectl exec $POD_NAME -n $NAMESPACE -- rm -f "$POD_BACKUP_PATH" 2>/dev/null || true + + log "Cleanup completed" +} + +# Show usage instructions +usage() { + show_banner + print_color "${YELLOW}" "Usage: $0" + echo + print_color "${CYAN}" "Description: This script is used to recover MySQL database for Euler Copilot project" + echo + print_color "${CYAN}" "Prerequisites:" + print_color "${WHITE}" " 1. kubectl configured and able to access cluster" + print_color "${WHITE}" " 2. mysql.sql file exists in current directory" + print_color "${WHITE}" " 3. Has sufficient cluster permissions" + echo + print_color "${RED}" "Warning: This operation will overwrite existing database data!" + echo +} + +# Confirm operation +confirm_operation() { + print_color "${YELLOW}" "Warning: This operation will clear existing database data and import new data!" + echo + read -p "$(print_color "${YELLOW}" "Confirm execution of database recovery operation? (y/N): ")" confirm + case $confirm in + [yY] | [yY][eE][sS]) + return 0 + ;; + *) + warning "Operation cancelled" + exit 0 + ;; + esac +} + +# Main function +main() { + show_banner + step "Starting MySQL database recovery process" + + check_dependencies + get_mysql_pod + get_mysql_password + check_backup_file + test_database_connection + confirm_operation + copy_backup_to_pod + restore_database + verify_restore + cleanup + + echo + print_color "${GREEN}" "================================================" + print_color "${GREEN}" " MySQL Data Recovery Completed!" + print_color "${GREEN}" "================================================" + echo + print_color "${GREEN}" "✓ Backup file check completed" + print_color "${GREEN}" "✓ Database connection test passed" + print_color "${GREEN}" "✓ Data recovery executed successfully" + print_color "${GREEN}" "✓ Temporary files cleanup completed" + echo + print_color "${BLUE}" "Tip: It is recommended to check application running status to ensure data recovery success." +} + +# Script entry point +if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then + usage + exit 0 +fi + +main diff --git a/deploy/en/scripts/9-other-script/migrate_opengauss_database.sh b/deploy/en/scripts/9-other-script/migrate_opengauss_database.sh new file mode 100755 index 000000000..e535bbea9 --- /dev/null +++ b/deploy/en/scripts/9-other-script/migrate_opengauss_database.sh @@ -0,0 +1,210 @@ +#!/bin/bash + +# OpenGauss Database Import Script +# Description: Used for Euler Copilot project database data import + +set -e # Exit immediately on error + +# Color definitions +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +PURPLE='\033[0;35m' +CYAN='\033[0;36m' +NC='\033[0m' # No Color + +# Print functions +log() { + echo -e "${GREEN}[$(date '+%Y-%m-%d %H:%M:%S')] $1${NC}" +} + +info() { + echo -e "${BLUE}[$(date '+%Y-%m-%d %H:%M:%S')] $1${NC}" +} + +warning() { + echo -e "${YELLOW}[$(date '+%Y-%m-%d %H:%M:%S')] Warning: $1${NC}" +} + +error() { + echo -e "${RED}[$(date '+%Y-%m-%d %H:%M:%S')] Error: $1${NC}" >&2 + exit 1 +} + +step() { + echo -e "${PURPLE}[$(date '+%Y-%m-%d %H:%M:%S')] === $1 ===${NC}" +} + +# Configuration variables +NAMESPACE="euler-copilot" +SECRET_NAME="euler-copilot-database" +PASSWORD_KEY="gauss-password" +BACKUP_FILE="/home/dump/opengauss/opengauss.sql" +POD_BACKUP_PATH="/home/omm/opengauss.sql" + +# Check required tools +check_dependencies() { + step "Checking required tools" + command -v kubectl >/dev/null 2>&1 || error "kubectl not installed" + command -v base64 >/dev/null 2>&1 || error "base64 not installed" + log "Dependency check passed" +} + +# Get database password +get_database_password() { + step "Getting database password" + PASSWORD=$(kubectl get secret $SECRET_NAME -n $NAMESPACE -o jsonpath="{.data.$PASSWORD_KEY}" 2>/dev/null | base64 --decode) + + if [ -z "$PASSWORD" ]; then + error "Unable to get database password, please check if secret exists" + fi + log "Password obtained successfully" +} + +# Get OpenGauss Pod name +get_opengauss_pod() { + step "Finding OpenGauss Pod" + POD_NAME=$(kubectl get pod -n $NAMESPACE 2>/dev/null | grep opengauss | grep Running | awk '{print $1}') + + if [ -z "$POD_NAME" ]; then + error "No running OpenGauss Pod found" + fi + log "Found Pod: $POD_NAME" +} + +# Check if backup file exists +check_backup_file() { + step "Checking backup file" + if [ ! -f "$BACKUP_FILE" ]; then + error "Backup file $BACKUP_FILE does not exist" + fi + log "Backup file check passed: $BACKUP_FILE" +} + +# Copy backup file to Pod +copy_backup_to_pod() { + step "Copying backup file to Pod" + kubectl cp "$BACKUP_FILE" "$POD_NAME:$POD_BACKUP_PATH" -n $NAMESPACE + kubectl exec -it "$POD_NAME" -n $NAMESPACE -- bash -c "chown omm:omm $POD_BACKUP_PATH" + log "Backup file copy completed" +} + +# Execute database import +import_database() { + step "Executing database import operation" + + info "Step 1: Disabling foreign key constraints..." + kubectl exec -it "$POD_NAME" -n $NAMESPACE -- su - omm -s /bin/bash -c \ + "gsql -d postgres -U postgres -W '$PASSWORD' -c \"SET session_replication_role = replica;\"" + log "Foreign key constraints disabled" + + info "Step 2: Clearing table data..." + kubectl exec -it "$POD_NAME" -n $NAMESPACE -- su - omm -s /bin/bash -c \ + "gsql -d postgres -U postgres -W '$PASSWORD' -c \" +TRUNCATE TABLE + action, team, knowledge_base, document, chunk, document_type, + role, role_action, users, task, task_report, team_user, user_role, + dataset, dataset_doc, image, qa, task_queue, team_message, + testcase, testing, user_message +CASCADE;\"" + log "Table data cleared" + + info "Step 3: Importing data..." + kubectl exec -it "$POD_NAME" -n $NAMESPACE -- su - omm -s /bin/bash -c \ + "gsql -d postgres -U postgres -f $POD_BACKUP_PATH -W '$PASSWORD'" + log "Data import completed" + + info "Step 4: Enabling foreign key constraints..." + kubectl exec -it "$POD_NAME" -n $NAMESPACE -- su - omm -s /bin/bash -c \ + "gsql -d postgres -U postgres -W '$PASSWORD' -c \"SET session_replication_role = origin;\"" + log "Foreign key constraints enabled" + + log "Database import operation fully completed" +} + +# Clean up temporary files +cleanup() { + step "Cleaning up temporary files" + kubectl exec -it "$POD_NAME" -n $NAMESPACE -- rm -f "$POD_BACKUP_PATH" 2>/dev/null || true + log "Cleanup completed" +} + +# Show banner +show_banner() { + echo -e "${PURPLE}" + echo "================================================================" + echo " OpenGauss Database Import Script" + echo " Euler Copilot Project Specific" + echo "================================================================" + echo -e "${NC}" +} + +# Main function +main() { + show_banner + step "Starting OpenGauss database import process" + + check_dependencies + get_database_password + get_opengauss_pod + check_backup_file + copy_backup_to_pod + import_database + cleanup + + echo -e "${GREEN}" + echo "================================================================" + echo " OpenGauss Data Import Completed!" + echo "================================================================" + echo -e "${NC}" + + echo -e "${GREEN}✓ Foreign key constraints disabled${NC}" + echo -e "${GREEN}✓ Table data cleared${NC}" + echo -e "${GREEN}✓ New data imported${NC}" + echo -e "${GREEN}✓ Foreign key constraints re-enabled${NC}" + echo "" + echo -e "${BLUE}Tip: It is recommended to check application running status to ensure data import success.${NC}" +} + +# Show usage instructions +usage() { + show_banner + echo -e "${YELLOW}Usage: $0${NC}" + echo "" + echo -e "Description: This script is used to import OpenGauss database data for Euler Copilot project" + echo "" + echo -e "${CYAN}Prerequisites:${NC}" + echo -e " 1. kubectl configured and able to access cluster" + echo -e " 2. opengauss.sql file exists in current directory" + echo -e " 3. Has sufficient cluster permissions" + echo "" + echo -e "${RED}Warning: This operation will clear existing database data and import new data!${NC}" +} + +# Script entry point +if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then + usage + exit 0 +fi + +# Fix color display issue +echo -e "${YELLOW}Warning: This operation will clear existing database data and import new data!${NC}" + +# Method 1: Use temporary variable +yellow_text="${YELLOW}Confirm execution of database import operation? (y/N): ${NC}" +read -p "$(echo -e "$yellow_text")" confirm + +# Method 2: Or use echo -e directly (alternative) +# echo -e "${YELLOW}Confirm execution of database import operation? (y/N): ${NC}\c" +# read confirm + +case $confirm in + [yY] | [yY][eE][sS]) + main + ;; + *) + warning "Operation cancelled" + exit 0 + ;; +esac diff --git a/deploy/en/scripts/9-other-script/modify_eulercopilot_yaml.py b/deploy/en/scripts/9-other-script/modify_eulercopilot_yaml.py new file mode 100755 index 000000000..856f51070 --- /dev/null +++ b/deploy/en/scripts/9-other-script/modify_eulercopilot_yaml.py @@ -0,0 +1,120 @@ +import sys +import argparse +from typing import Union + +# Version marker and dependency detection +try: + from ruamel.yaml import YAML + from ruamel.yaml.comments import CommentedMap + USING_RUAMEL = True +except ImportError: + try: + import yaml # PyYAML fallback + USING_RUAMEL = False + except ImportError as e: + sys.stderr.write("Error: YAML processing library required\n") + sys.stderr.write("Please install one of the following:\n") + sys.stderr.write("1. (Recommended) ruamel.yaml: pip install ruamel.yaml\n") + sys.stderr.write("2. PyYAML: pip install PyYAML\n") + sys.exit(1) + +def parse_value(value: str) -> Union[str, int, float, bool]: + """Intelligently convert value types""" + value = value.strip() + lower_val = value.lower() + + if lower_val in {'true', 'false'}: + return lower_val == 'true' + if lower_val in {'null', 'none'}: + return None + + try: + return int(value) + except ValueError: + try: + return float(value) + except ValueError: + pass + + # Handle quoted strings + if len(value) > 1 and value[0] == value[-1] and value[0] in {'"', "'"}: + return value[1:-1] + + return value + +def set_nested_value(data: dict, key_path: str, value: str) -> None: + """Recursively set nested dictionary values""" + keys = key_path.split('.') + current = data + + try: + for key in keys[:-1]: + # Automatically create missing levels + if key not in current: + current[key] = CommentedMap() if USING_RUAMEL else {} + current = current[key] + + final_key = keys[-1] + current[final_key] = parse_value(value) + except TypeError as e: + raise ValueError(f"Non-dictionary type intermediate node found in path {key_path}") from e + +def main(): + parser = argparse.ArgumentParser( + description='YAML configuration file modification tool', + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument('input', help='Input YAML file path') + parser.add_argument('output', help='Output YAML file path') + parser.add_argument('--set', + action='append', + required=True, + help='Format: path.to.key=value (can be used multiple times)', + metavar='KEY_PATH=VALUE') + + args = parser.parse_args() + + # Initialize YAML processor + if USING_RUAMEL: + yaml_processor = YAML() + yaml_processor.preserve_quotes = True + yaml_processor.indent(mapping=2, sequence=4, offset=2) + else: + yaml_processor = yaml # Use PyYAML module + + # Read file (corrected section) + try: + with open(args.input, 'r') as f: # Ensure this line is properly closed + if USING_RUAMEL: + data = yaml_processor.load(f) + else: + data = yaml.safe_load(f) + except Exception as e: + raise SystemExit(f"Failed to read file: {str(e)}") + + # Process modification parameters + for item in args.set: + if '=' not in item: + raise ValueError(f"Invalid format: {item}, should use KEY_PATH=VALUE format") + + key_path, value = item.split('=', 1) + if not key_path: + raise ValueError("Key path cannot be empty") + + try: + set_nested_value(data, key_path, value) + except Exception as e: + raise SystemExit(f"Error setting {key_path}: {str(e)}") + + # Write file + try: + with open(args.output, 'w') as f: + if USING_RUAMEL: + yaml_processor.dump(data, f) + else: + yaml.dump(data, f, default_flow_style=False, indent=2) + except Exception as e: + raise SystemExit(f"Failed to write file: {str(e)}") + +if __name__ == '__main__': + main() diff --git a/deploy/en/scripts/9-other-script/prepare_docker.sh b/deploy/en/scripts/9-other-script/prepare_docker.sh new file mode 100755 index 000000000..4b429dc73 --- /dev/null +++ b/deploy/en/scripts/9-other-script/prepare_docker.sh @@ -0,0 +1,163 @@ +#!/bin/bash + +function stop_docker { + echo -e "[Info] Checking if Docker is installed"; + if ! [[ -x $(command -v docker) ]]; then + echo -e "[Info] Docker not installed"; + return 0; + fi + + echo -e "\033[33m[Warning] About to stop Docker service, confirm to continue?\033[0m"; + read -p "(Y/n): " choice; + case $choice in + [Yy]) + systemctl stop docker + if [[ $? -ne 0 ]]; then + echo -e "\033[31m[Error] Failed to stop Docker service, aborting\033[0m" + return 1 + else + echo -e "\033[32m[Success] Docker service stopped successfully\033[0m" + fi + ;; + [Nn]) + echo -e "\033[31m[Error] Operation cancelled\033[0m" + return 1 + ;; + *) + echo -e "\033[31m[Error] Invalid input, operation cancelled\033[0m" + return 1 + ;; + esac + + echo -e "\033[33m[Warning] About to attempt to uninstall old Docker version, confirm to continue?\033[0m"; + read -p "(Y/n): " choice2; + case $choice2 in + [Yy]) + yum remove -y docker-client docker-client-latest docker-common docker-latest docker-latest-logrotate docker-logrotate docker-engine + if [[ $? -ne 0 ]]; then + echo -e "\033[31m[Error] Failed to uninstall old Docker version\033[0m" + return 1 + else + echo -e "\033[32m[Success] Old Docker version uninstalled\033[0m" + fi + ;; + [Nn]) + echo -e "\033[31m[Error] Operation cancelled\033[0m" + return 1 + ;; + *) + echo -e "\033[31m[Error] Invalid input, operation cancelled\033[0m" + return 1 + ;; + esac + return 0; +} + +function setup_docker_repo { + echo -e "[Info] Setting up Docker RPM Repo"; + basearch=$(arch) + cat > /etc/yum.repos.d/docker-ce.repo <<-EOF +[docker-ce-stable] +name=Docker CE Stable - \$basearch +baseurl=https://mirrors.tuna.tsinghua.edu.cn/docker-ce/linux/centos/9/\$basearch/stable +enabled=1 +gpgcheck=1 +gpgkey=https://mirrors.tuna.tsinghua.edu.cn/docker-ce/linux/centos/gpg +EOF + echo -e "[Info] Updating yum package list"; + yum makecache + if [[ $? -ne 0 ]]; then + echo -e "\033[31m[Error] Failed to update yum package list\033[0m"; + return 1; + else + echo -e "\033[32m[Success] yum package list updated successfully\033[0m"; + fi + return 0; +} + +function install_docker { + echo -e "[Info] Installing Docker"; + yum install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin; + if [[ $? -ne 0 ]]; then + echo -e "\033[31m[Error] Failed to install Docker\033[0m"; + return 1; + else + echo -e "\033[32m[Success] Docker installed successfully\033[0m"; + fi + systemctl enable docker; + + echo -e "[Info] Setting up DockerHub mirror"; + if ! [[ -d "/etc/docker" ]]; then + mkdir /etc/docker; + fi + + if [[ -f "/etc/docker/daemon.json" ]]; then + echo -e "\033[31m[Error] daemon.json already exists, please manually configure DockerHub mirror\033[0m"; + else + cat > /etc/docker/daemon.json <<-EOF +{ + "registry-mirrors": [ + "https://docker.anyhub.us.kg", + "https://docker.1panel.live", + "https://dockerhub.icu", + "https://docker.ckyl.me", + "https://docker.awsl9527.cn", + "https://dhub.kubesre.xyz", + "https://gg3gwnry.mirror.aliyuncs.com" + ] +} +EOF + fi + systemctl restart docker; + if [[ $? -ne 0 ]]; then + echo -e "\033[31m[Error] Docker startup failed\033[0m"; + return 1; + else + echo -e "\033[32m[Success] Docker started successfully\033[0m"; + return 0; + fi +} + +function login_docker { + echo -e "[Info] Logging into Docker private registry"; + read -p "Registry URL: " url; + read -p "Username: " username; + read -p "Password: " password; + + docker login -u $username -p $password $url; + if [[ $? -ne 0 ]]; then + echo -e "\033[31m[Error] Docker login failed\033[0m"; + return 1; + else + echo -e "\033[32m[Success] Docker login successful\033[0m"; + return 0; + fi +} + +function main { + echo -e "[Info] Updating Docker"; + + stop_docker; + if [[ $? -ne 0 ]]; then + return 1; + fi + + setup_docker_repo; + if [[ $? -ne 0 ]]; then + return 1; + fi + + install_docker; + if [[ $? -ne 0 ]]; then + return 1; + fi + + login_docker; + if [[ $? -ne 0 ]]; then + return 1; + fi + + return 0; +} + +main diff --git a/deploy/en/scripts/9-other-script/save_images.sh b/deploy/en/scripts/9-other-script/save_images.sh new file mode 100755 index 000000000..000e0b58b --- /dev/null +++ b/deploy/en/scripts/9-other-script/save_images.sh @@ -0,0 +1,187 @@ +#!/bin/bash + +# Color definitions +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # Reset color + +# Default configuration +eulercopilot_version="0.10.0" +ARCH_SUFFIX="" +OUTPUT_DIR="/home/eulercopilot/images/${eulercopilot_version}" + +# Show help information +show_help() { + echo -e "${YELLOW}Usage:${NC}" + echo -e " $0 [options]" + echo -e "" + echo -e "${YELLOW}Options:${NC}" + echo -e " --help Show this help message" + echo -e " --version Specify EulerCopilot version (default: ${eulercopilot_version})" + echo -e " --arch Specify system architecture (arm/x86, default: auto-detect)" + echo -e "" + echo -e "${YELLOW}Examples:${NC}" + echo -e " $0 --version ${eulercopilot_version} --arch arm" + echo -e " $0 --help" + exit 0 +} + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case "$1" in + --help) + show_help + ;; + --version) + if [ -n "$2" ]; then + eulercopilot_version="$2" + OUTPUT_DIR="/home/eulercopilot/images/${eulercopilot_version}" + shift + else + echo -e "${RED}Error: --version requires a version number${NC}" + exit 1 + fi + ;; + --arch) + if [ -n "$2" ]; then + case "$2" in + arm|x86) + ARCH_SUFFIX="$2" + ;; + *) + echo -e "${RED}Error: Unsupported architecture '$2', must be arm or x86${NC}" + exit 1 + ;; + esac + shift + else + echo -e "${RED}Error: --arch requires an architecture (arm/x86)${NC}" + exit 1 + fi + ;; + *) + echo -e "${RED}Unknown parameter: $1${NC}" + show_help + ;; + esac + shift +done + +# Auto-detect architecture (if not specified via parameter) +if [ -z "$ARCH_SUFFIX" ]; then + ARCH=$(uname -m) + case $ARCH in + x86_64) + ARCH_SUFFIX="x86" + ;; + aarch64|armv*) + ARCH_SUFFIX="arm" + ;; + *) + echo -e "${RED}Unsupported architecture: $ARCH${NC}" + exit 1 + ;; + esac +fi + +mkdir -p "$OUTPUT_DIR" + +# Image list (using version variable) +BASE_IMAGES=( + "hub.oepkgs.net/neocopilot/euler-copilot-framework:${eulercopilot_version}-arm" + "hub.oepkgs.net/neocopilot/euler-copilot-web:${eulercopilot_version}-arm" + "hub.oepkgs.net/neocopilot/data_chain_back_end:${eulercopilot_version}-arm" + "hub.oepkgs.net/neocopilot/data_chain_web:${eulercopilot_version}-arm" + "hub.oepkgs.net/neocopilot/authhub:0.9.3-arm" + "hub.oepkgs.net/neocopilot/authhub-web:0.9.3-arm" + "hub.oepkgs.net/neocopilot/opengauss:latest-arm" + "hub.oepkgs.net/neocopilot/redis:7.4-alpine-arm" + "hub.oepkgs.net/neocopilot/mysql:8-arm" + "hub.oepkgs.net/neocopilot/minio:empty-arm" + "hub.oepkgs.net/neocopilot/mongo:7.0.16-arm" + "hub.oepkgs.net/neocopilot/secret_inject:dev-arm" +) + +# Predefined filename list (strictly corresponding to BASE_IMAGES order) +FILE_NAMES=( + "euler-copilot-framework.tar" + "euler-copilot-web.tar" + "data_chain_back_end.tar" + "data_chain_web.tar" + "authhub.tar" + "authhub-web.tar" + "opengauss.tar" + "redis.tar" + "mysql.tar" + "minio.tar" + "mongo.tar" + "secret_inject.tar" +) + +# Validate list consistency +if [ ${#BASE_IMAGES[@]} -ne ${#FILE_NAMES[@]} ]; then + echo -e "${RED}Error: Image list and filename list count mismatch${NC}" + exit 1 +fi + +# Initialize counters +total=${#BASE_IMAGES[@]} +success=0 +fail=0 + +# Image processing function +process_image() { + local raw_image=$1 + local filename=$2 + + # Adjust architecture tag + local adjusted_image="${raw_image/-arm/-${ARCH_SUFFIX}}" + local output_path="${OUTPUT_DIR}/${filename}" + + echo -e "\n${BLUE}Processing: ${adjusted_image}${NC}" + + # Pull image + if ! docker pull "$adjusted_image"; then + echo -e "${RED}Pull failed: ${adjusted_image}${NC}" + return 1 + fi + + # Save image + if docker save -o "$output_path" "$adjusted_image"; then + echo -e "${GREEN}Image saved to: ${output_path}${NC}" + return 0 + else + echo -e "${RED}Save failed: ${output_path}${NC}" + return 1 + fi +} + +# Print execution information +echo -e "${BLUE}==============================${NC}" +echo -e "${YELLOW}Architecture\t: ${ARCH_SUFFIX}${NC}" +echo -e "${YELLOW}Version\t\t: ${eulercopilot_version}${NC}" +echo -e "${YELLOW}Storage directory\t: ${OUTPUT_DIR}${NC}" +echo -e "${YELLOW}Image count\t: ${total}${NC}" +echo -e "${BLUE}==============================${NC}" + +# Process all images +for index in "${!BASE_IMAGES[@]}"; do + if process_image "${BASE_IMAGES[$index]}" "${FILE_NAMES[$index]}"; then + ((success++)) + else + ((fail++)) + fi +done + +# Output final result +echo -e "\n${BLUE}==============================${NC}" +echo -e "${GREEN}Operation completed!${NC}" +echo -e "${BLUE}==============================${NC}" +echo -e "${GREEN}Success\t: ${success}${NC}" +echo -e "${RED}Failed\t: ${fail}${NC}" +echo -e "${BLUE}==============================${NC}" + +# Return status code +exit $((fail > 0 ? 1 : 0)) diff --git a/deploy/en/scripts/deploy.sh b/deploy/en/scripts/deploy.sh new file mode 100755 index 000000000..0a1dd7977 --- /dev/null +++ b/deploy/en/scripts/deploy.sh @@ -0,0 +1,329 @@ +#!/bin/bash + +# Top level menu +show_top_menu() { + clear + echo "==============================" + echo " Main Deployment Menu " + echo "==============================" + echo "0) One-click Auto Deployment" + echo "1) Manual Step-by-Step Deployment" + echo "2) Restart Services" + echo "3) Uninstall All Components and Clear Data" + echo "4) Exit Program" + echo "==============================" + echo -n "Enter option number (0-4): " +} + +# Installation options menu (manual deployment submenu) +show_sub_menu() { + clear + echo "==============================" + echo " Manual Step-by-Step Deployment Menu" + echo "==============================" + echo "1) Execute Environment Check Script" + echo "2) Install k3s and helm" + echo "3) Install Ollama" + echo "4) Deploy Deepseek Model" + echo "5) Deploy Embedding Model" + echo "6) Install Databases" + echo "7) Install AuthHub" + echo "8) Install EulerCopilot" + echo "9) Return to Main Menu" + echo "==============================" + echo -n "Enter option number (1-9): " +} + +show_restart_menu() { + clear + echo "==============================" + echo " Service Restart Menu " + echo "==============================" + echo "Available services to restart:" + echo "1) authhub-backend" + echo "2) authhub" + echo "3) framework" + echo "4) minio" + echo "5) mongo" + echo "6) mysql" + echo "7) opengauss" + echo "8) rag" + echo "9) rag-web" + echo "10) redis" + echo "11) web" + echo "12) Return to Main Menu" + echo "==============================" + echo -n "Enter service number to restart (1-12): " +} + +# Script execution function with error checking +run_script_with_check() { + local script_path=$1 + local script_name=$2 + + echo "--------------------------------------------------" + echo "Starting: $script_name" + "$script_path" || { + echo -e "\n\033[31m$script_name execution failed!\033[0m" + exit 1 + } + echo -e "\n\033[32m$script_name executed successfully!\033[0m" + echo "--------------------------------------------------" +} + +# Execute submenu corresponding scripts +run_sub_script() { + case $1 in + 1) + run_script_with_check "./1-check-env/check_env.sh" "Environment Check Script" + ;; + 2) + run_script_with_check "./2-install-tools/install_tools.sh" "k3s and helm Installation Script" + ;; + 3) + run_script_with_check "./3-install-ollama/install_ollama.sh" "Ollama Installation Script" + ;; + 4) + run_script_with_check "./4-deploy-deepseek/deploy_deepseek.sh" "Deepseek Deployment Script" + ;; + 5) + run_script_with_check "./5-deploy-embedding/deploy-embedding.sh" "Embedding Deployment Script" + ;; + 6) + run_script_with_check "./6-install-databases/install_databases.sh" "Database Installation Script" + ;; + 7) + run_script_with_check "./7-install-authhub/install_authhub.sh" "AuthHub Installation Script" + ;; + 8) + run_script_with_check "./8-install-EulerCopilot/install_eulercopilot.sh" "EulerCopilot Installation Script" + ;; + 9) + echo "Returning to main menu..." + echo "Press any key to continue..." + read -r -n 1 -s + return 2 # Special return code for returning to parent menu + ;; + *) + echo -e "\033[31mInvalid option, please enter a number between 1-9\033[0m" + return 1 + ;; + esac + return 0 +} + +# Uninstall all components +uninstall_all() { + echo -e "\033[31mWarning: This operation will permanently delete all components and data!\033[0m" + read -p "Are you sure you want to continue? (y/n) " confirm + + if [[ $confirm != "y" && $confirm != "Y" ]]; then + echo "Uninstall operation cancelled" + return + fi + + # Set timeout (in seconds) + local HELM_TIMEOUT=300 + local PVC_DELETE_TIMEOUT=120 + local FORCE_DELETE=false + + echo "Starting to uninstall all Helm Releases..." + local RELEASES + RELEASES=$(helm list -n euler-copilot --short) + + # Delete all associated Helm Releases + if [ -n "$RELEASES" ]; then + echo -e "${YELLOW}Found the following Helm Releases, starting cleanup...${NC}" + for release in $RELEASES; do + echo -e "${BLUE}Deleting Helm Release: ${release}${NC}" + if ! helm uninstall "$release" -n euler-copilot \ + --wait \ + --timeout ${HELM_TIMEOUT}s \ + --no-hooks; then + echo -e "${RED}Warning: Helm Release ${release} deletion abnormal, attempting force delete...${NC}" + FORCE_DELETE=true + helm uninstall "$release" -n euler-copilot \ + --timeout 10s \ + --no-hooks \ + --force || true + fi + done + else + echo -e "${YELLOW}No Helm Releases found to clean${NC}" + fi + + # Wait for resource release + sleep 10 + + # Get all PVC list + local pvc_list + pvc_list=$(kubectl get pvc -n euler-copilot -o name 2>/dev/null) + + # Delete PVC (with retry mechanism) + if [ -n "$pvc_list" ]; then + echo -e "${YELLOW}Found the following PVCs, starting cleanup...${NC}" + local start_time=$(date +%s) + local end_time=$((start_time + PVC_DELETE_TIMEOUT)) + + for pvc in $pvc_list; do + while : ; do + # Attempt normal deletion + if kubectl delete $pvc -n euler-copilot --timeout=30s 2>/dev/null; then + break + fi + + # Check if timeout + if [ $(date +%s) -ge $end_time ]; then + echo -e "${RED}Error: PVC deletion timeout, attempting force cleanup...${NC}" + + # Remove Finalizer force delete + kubectl patch $pvc -n euler-copilot \ + --type json \ + --patch='[ { "op": "remove", "path": "/metadata/finalizers" } ]' 2>/dev/null || true + + # Force delete + kubectl delete $pvc -n euler-copilot \ + --force \ + --grace-period=0 2>/dev/null && break || true + + # Final confirmation + if ! kubectl get $pvc -n euler-copilot &>/dev/null; then + break + fi + echo -e "${RED}Critical Error: Unable to delete PVC ${pvc}${NC}" >&2 + return 1 + fi + + # Wait and retry + sleep 5 + echo -e "${YELLOW}Retrying to delete PVC: ${pvc}...${NC}" + done + done + else + echo -e "${YELLOW}No PVCs found to clean${NC}" + fi + + # Delete specified Secrets + local secret_list=("authhub-secret" "euler-copilot-database" "euler-copilot-system") + for secret in "${secret_list[@]}"; do + if kubectl get secret "$secret" -n euler-copilot &>/dev/null; then + echo -e "${YELLOW}Found Secret: ${secret}, starting cleanup...${NC}" + if ! kubectl delete secret "$secret" -n euler-copilot; then + echo -e "${RED}Error: Failed to delete Secret ${secret}!${NC}" >&2 + return 1 + fi + else + echo -e "${YELLOW}No Secret found to clean: ${secret}${NC}" + fi + done + + # Final cleanup check + echo -e "${YELLOW}Performing final resource check...${NC}" + kubectl delete all --all -n euler-copilot --timeout=30s 2>/dev/null || true + + echo -e "${GREEN}Resource cleanup completed${NC}" + echo -e "\033[32mAll components and data have been successfully cleared\033[0m" +} + +# Manual deployment submenu loop +manual_deployment_loop() { + while true; do + show_sub_menu + read -r sub_choice + run_sub_script "$sub_choice" + retval=$? + + if [ $retval -eq 2 ]; then # Return to main menu + break + elif [ $retval -eq 0 ]; then + echo "Press any key to continue..." + read -r -n 1 -s + fi + done +} + +restart_pod() { + local service="$1" + if [[ -z "$service" ]]; then + echo -e "${RED}Error: Please enter service name${NC}" + return 1 + fi + + local deployment="${service}-deploy" + echo -e "${BLUE}Verifying if deployment exists...${NC}" + if ! kubectl get deployment "$deployment" -n euler-copilot &> /dev/null; then + echo -e "${RED}Error: Could not find deployment $deployment in euler-copilot namespace${NC}" + return 1 + fi + + echo -e "${YELLOW}Restarting deployment $deployment ...${NC}" + if kubectl rollout restart deployment/"$deployment" -n euler-copilot; then + echo -e "${GREEN}Successfully triggered rolling restart!${NC}" + echo -e "You can check status with:\nkubectl rollout status deployment/$deployment -n euler-copilot" + return 0 + else + echo -e "${RED}Failed to restart deployment $deployment!${NC}" + return 1 + fi +} + +# Main program loop improved +while true; do + show_top_menu + read -r main_choice + + case $main_choice in + 0) + run_script_with_check "./0-one-click-deploy/one-click-deploy.sh" "One-click Auto Deployment" + echo "Press any key to continue..." + read -r -n 1 -s + ;; + 1) + manual_deployment_loop + ;; + 2) + while true; do + show_restart_menu + read -r restart_choice + case $restart_choice in + 1) service="authhub-backend" ;; + 2) service="authhub" ;; + 3) service="framework" ;; + 4) service="minio" ;; + 5) service="mongo" ;; + 6) service="mysql" ;; + 7) service="opengauss" ;; + 8) service="rag" ;; + 9) service="rag-web" ;; + 10) service="redis" ;; + 11) service="web" ;; + 12) break ;; + *) + echo -e "${RED}Invalid option, please enter a number between 1-12${NC}" + continue + ;; + esac + + if [[ -n "$service" ]]; then + restart_pod "$service" + echo "Press any key to continue..." + read -r -n 1 -s + fi + done + ;; + + 3) + uninstall_all + echo "Press any key to continue..." + read -r -n 1 -s + ;; + 4) + echo "Exiting deployment system" + exit 0 + ;; + *) + echo -e "${RED}Invalid option, please enter a number between 0-4${NC}" + sleep 1 + ;; + esac +done diff --git a/deploy/en/secret_helper/Dockerfile b/deploy/en/secret_helper/Dockerfile new file mode 100644 index 000000000..2fe4fe7f6 --- /dev/null +++ b/deploy/en/secret_helper/Dockerfile @@ -0,0 +1,14 @@ +FROM hub.oepkgs.net/openeuler/openeuler:22.03-lts-sp4 +RUN mkdir /app && \ + mkdir /secrets +WORKDIR /app +COPY . . +RUN sed -i 's|repo.openeuler.org|mirrors.nju.edu.cn/openeuler|g' /etc/yum.repos.d/openEuler.repo && \ + sed -i '/metalink/d' /etc/yum.repos.d/openEuler.repo && \ + sed -i '/metadata_expire/d' /etc/yum.repos.d/openEuler.repo && \ + yum update -y && \ + yum install python3 python3-pip -y && \ + yum clean all && \ + pip3 install --no-cache-dir -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple +ENV PYTHONPATH=/app +ENTRYPOINT ["python3", "./main.py"] \ No newline at end of file diff --git a/deploy/en/secret_helper/__init__.py b/deploy/en/secret_helper/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/deploy/en/secret_helper/config.example.yaml b/deploy/en/secret_helper/config.example.yaml new file mode 100644 index 000000000..bde10ed50 --- /dev/null +++ b/deploy/en/secret_helper/config.example.yaml @@ -0,0 +1,14 @@ +copy: + - from: /config + to: /config-rw + mode: + uid: 1000 + gid: 1000 + mode: "0o750" + + - from: /etc/config + to: /config-rw/secret + mode: + uid: 1000 + gid: 1000 + mode: "0o750" diff --git a/deploy/en/secret_helper/file_copy.py b/deploy/en/secret_helper/file_copy.py new file mode 100644 index 000000000..ed3466457 --- /dev/null +++ b/deploy/en/secret_helper/file_copy.py @@ -0,0 +1,86 @@ +""" +Copy files and directories + +Copyright (c) Huawei Technologies Co., Ltd. 2023-2025. All rights reserved. +""" + +import logging +import os +from pathlib import Path +from typing import Any + +logger = logging.getLogger(__name__) + +def chown_chmod(path: Path, mode_number: int, uid: int, gid: int) -> None: + """改变文件权限""" + os.chown(str(path), uid, gid) # type: ignore[attr-defined] + path.chmod(mode_number) + + for file in path.rglob("*"): + os.chown(str(file), uid, gid) # type: ignore[attr-defined] + file.chmod(mode_number) + +def copy_file(file: Path, out_path: Path, secrets: dict[str, str]) -> None: + """复制单个文件""" + logger.info("复制文件: %s to %s", file, out_path) + with file.open("r", encoding="utf-8") as f: + data = f.read() + if secrets: + for key, value in secrets.items(): + data = data.replace(r"${" + key + "}", value) + # 确保父文件夹存在 + out_path.parent.mkdir(parents=True, exist_ok=True) + with out_path.open("w", encoding="utf-8") as f: + f.write(data) + +def process_item(from_path: Path, to_path: Path, secrets: dict[str, str]) -> None: + """处理Config中的单个项""" + if from_path.is_file(): + logger.info("找到文件: %s", from_path) + copy_file(from_path, to_path, secrets) + + for file in from_path.rglob("*"): + logger.info("找到文件: %s", file) + if any(p for p in file.parts if p.startswith(".")): + logger.info("跳过文件: %s", file) + continue + out_path = to_path / file.relative_to(from_path) + if file.is_file(): + copy_file(file, out_path, secrets) + else: + out_path.mkdir(parents=True, exist_ok=True) + + +def copy(from_path_str: str, to_path_str: str, mode: dict[str, Any], secrets_dir: list[str]) -> None: + """复制文件和目录""" + # 校验Secrets是否存在 + secrets = {} + for dir_name in secrets_dir: + secrets_path = Path(dir_name) + if not secrets_path.exists(): + continue + if not secrets_path.is_dir(): + err = f"{dir_name} 不是文件夹" + raise FileNotFoundError(err) + + # 读取secrets + for secret in secrets_path.iterdir(): + if secret.name.startswith(".") or secret.name.startswith("_") or not secret.is_file(): + continue + with secret.open("r") as f: + secrets[secret.name] = f.read() + + # 检查文件位置 + from_path = Path(from_path_str) + to_path = Path(to_path_str) + + # 检查文件是否存在 + if not from_path.exists(): + raise FileNotFoundError + + # 递归复制文件 + process_item(from_path, to_path, secrets) + + # 设置权限 + mode_number = int(mode["mode"], 8) + chown_chmod(to_path, mode_number, mode["uid"], mode["gid"]) diff --git a/deploy/en/secret_helper/job.py b/deploy/en/secret_helper/job.py new file mode 100644 index 000000000..bdd243226 --- /dev/null +++ b/deploy/en/secret_helper/job.py @@ -0,0 +1,11 @@ +""" +自动处理启动和停止 + +Copyright (c) Huawei Technologies Co., Ltd. 2023-2025. All rights reserved. +""" + +import pykube + + +def job() -> None: + pass diff --git a/deploy/en/secret_helper/main.py b/deploy/en/secret_helper/main.py new file mode 100644 index 000000000..a810ea5a5 --- /dev/null +++ b/deploy/en/secret_helper/main.py @@ -0,0 +1,41 @@ +""" +注入Secret + +Copyright (c) Huawei Technologies Co., Ltd. 2023-2025. All rights reserved. +""" + +import argparse +import logging +import sys +from pathlib import Path + +import yaml +from file_copy import copy +from job import job + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + logger = logging.getLogger(__name__) + + parser = argparse.ArgumentParser() + parser.add_argument("--config", type=str, default="config.yaml") + parser.add_argument("--copy", action="store_true") + parser.add_argument("--job", action="store_true") + parser.add_argument("--install", action="store_true") + args = parser.parse_args() + + if args.job: + job() + sys.exit(0) + + if args.copy: + with Path(args.config).open("r") as f: + config = yaml.safe_load(f) + + for copy_config in config["copy"]: + secrets = copy_config.get("secrets", []) + copy(copy_config["from"], copy_config["to"], copy_config["mode"], secrets) + + sys.exit(0) + + logger.error("Invalid argument") diff --git a/deploy/en/secret_helper/requirements.txt b/deploy/en/secret_helper/requirements.txt new file mode 100644 index 000000000..425af8d85 --- /dev/null +++ b/deploy/en/secret_helper/requirements.txt @@ -0,0 +1 @@ +pykube-ng==23.6.0 \ No newline at end of file -- Gitee