From 91fcd30438d9af4a259ede8f2fe68922d649647d Mon Sep 17 00:00:00 2001 From: z30057876 Date: Tue, 11 Feb 2025 20:23:45 +0800 Subject: [PATCH 1/3] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=92=8C=E5=BC=80?= =?UTF-8?q?=E6=94=BESecret=E5=A4=8D=E5=88=B6=E5=B7=A5=E5=85=B7=E4=BB=A3?= =?UTF-8?q?=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deploy/secret_helper/Dockerfile | 11 ++++ deploy/secret_helper/__init__.py | 0 deploy/secret_helper/config.example.yaml | 14 +++++ deploy/secret_helper/file_copy.py | 67 ++++++++++++++++++++++++ deploy/secret_helper/job.py | 9 ++++ deploy/secret_helper/main.py | 22 ++++++++ deploy/secret_helper/requirements.txt | 1 + 7 files changed, 124 insertions(+) create mode 100644 deploy/secret_helper/Dockerfile create mode 100644 deploy/secret_helper/__init__.py create mode 100644 deploy/secret_helper/config.example.yaml create mode 100644 deploy/secret_helper/file_copy.py create mode 100644 deploy/secret_helper/job.py create mode 100644 deploy/secret_helper/main.py create mode 100644 deploy/secret_helper/requirements.txt diff --git a/deploy/secret_helper/Dockerfile b/deploy/secret_helper/Dockerfile new file mode 100644 index 000000000..ca8421e35 --- /dev/null +++ b/deploy/secret_helper/Dockerfile @@ -0,0 +1,11 @@ +FROM hub.oepkgs.net/openeuler/openeuler:22.03-lts-sp4 +RUN mkdir /app && \ + mkdir /secrets +WORKDIR /app +COPY . . +RUN yum update -y && \ + yum install python3 python3-pip -y && \ + yum clean all && \ + pip3 install --no-cache-dir -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple +ENV PYTHONPATH=/app +ENTRYPOINT ["python3", "./main.py"] \ No newline at end of file diff --git a/deploy/secret_helper/__init__.py b/deploy/secret_helper/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/deploy/secret_helper/config.example.yaml b/deploy/secret_helper/config.example.yaml new file mode 100644 index 000000000..bde10ed50 --- /dev/null +++ b/deploy/secret_helper/config.example.yaml @@ -0,0 +1,14 @@ +copy: + - from: /config + to: /config-rw + mode: + uid: 1000 + gid: 1000 + mode: "0o750" + + - from: /etc/config + to: /config-rw/secret + mode: + uid: 1000 + gid: 1000 + mode: "0o750" diff --git a/deploy/secret_helper/file_copy.py b/deploy/secret_helper/file_copy.py new file mode 100644 index 000000000..dd4161857 --- /dev/null +++ b/deploy/secret_helper/file_copy.py @@ -0,0 +1,67 @@ +"""Copy files and directories + +Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. +""" +import os +from pathlib import Path +from typing import Any + + +def chown_chmod(path: Path, mode_number: int, uid: int, gid: int) -> None: + """Change ownership and permissions""" + path.chmod(mode_number) + os.chown(str(path), uid, gid) # type: ignore[] + + for file in path.rglob("*"): + os.chown(str(file), uid, gid) # type: ignore[] + file.chmod(mode_number) + + +def copy_single_file(from_path: Path, to_path: Path, secrets: dict[str, str]) -> None: + """Copy a single file""" + for file in from_path.rglob("*"): + print(f"found: {file}") + if any(p for p in file.parts if p.startswith(".")): + print(f"skipping: {file}") + continue + out_path = to_path / file.relative_to(from_path) + if file.is_file(): + print(f"copying: {file} to {out_path}") + with file.open("r", encoding="utf-8") as f: + data = f.read() + if secrets: + for key, value in secrets.items(): + data = data.replace(r"${" + key + "}", value) + with out_path.open("w", encoding="utf-8") as f: + f.write(data) + else: + out_path.mkdir(parents=True, exist_ok=True) + + +def copy(from_path_str: str, to_path_str: str, mode: dict[str, Any]) -> None: + """Copy files and directories""" + # 校验Secrets是否存在 + secrets_path = Path("/secrets") + if not secrets_path.exists(): + secrets = {} + else: + # 读取secrets + secrets = {} + for secret in secrets_path.iterdir(): + with secret.open("r") as f: + secrets[secret.name] = f.read() + + # 检查文件位置 + from_path = Path(from_path_str) + to_path = Path(to_path_str) + + # 检查文件是否存在 + if not from_path.exists(): + raise FileNotFoundError + + # 递归复制文件 + copy_single_file(from_path, to_path, secrets) + + # 设置权限 + mode_number = int(mode["mode"], 8) + chown_chmod(to_path, mode_number, mode["uid"], mode["gid"]) diff --git a/deploy/secret_helper/job.py b/deploy/secret_helper/job.py new file mode 100644 index 000000000..928be73e1 --- /dev/null +++ b/deploy/secret_helper/job.py @@ -0,0 +1,9 @@ +"""Recreate failed pods + +Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. +""" +import pykube + + +def job() -> None: + pass diff --git a/deploy/secret_helper/main.py b/deploy/secret_helper/main.py new file mode 100644 index 000000000..be90de10d --- /dev/null +++ b/deploy/secret_helper/main.py @@ -0,0 +1,22 @@ +"""Secret Injector + +Copyright (c) Huawei Technologies Co., Ltd. 2023-2024. All rights reserved. +""" +from pathlib import Path + +import yaml + +from file_copy import copy +from job import job + +if __name__ == "__main__": + config = Path("config.yaml") + if not config.exists(): + job() + + else: + with config.open("r") as f: + config = yaml.safe_load(f) + + for copy_config in config["copy"]: + copy(copy_config["from"], copy_config["to"], copy_config["mode"]) diff --git a/deploy/secret_helper/requirements.txt b/deploy/secret_helper/requirements.txt new file mode 100644 index 000000000..425af8d85 --- /dev/null +++ b/deploy/secret_helper/requirements.txt @@ -0,0 +1 @@ +pykube-ng==23.6.0 \ No newline at end of file -- Gitee From 298328df8c9e8d169bf719091ba3052f3e0a40d2 Mon Sep 17 00:00:00 2001 From: z30057876 Date: Tue, 11 Feb 2025 20:24:42 +0800 Subject: [PATCH 2/3] =?UTF-8?q?=E5=AE=9E=E7=8E=B0authhub=E4=B8=80=E9=94=AE?= =?UTF-8?q?=E9=83=A8=E7=BD=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../authhub/configs/backend/aops-config.yml | 10 +- .../authhub/configs/backend/copy-config.yml | 7 ++ .../authhub/configs/web/authhub.nginx.conf | 35 ------ ...ecret.yaml => authhub-backend-config.yaml} | 9 +- .../backend/authhub-backend-deployment.yaml | 51 -------- .../backend/authhub-backend-service.yaml | 17 --- .../templates/backend/authhub-backend.yaml | 91 ++++++++++++++ .../{mysql-secret.yaml => mysql-config.yaml} | 10 +- .../templates/mysql/mysql-service.yaml | 17 --- .../{mysql-pvc.yaml => mysql-storage.yaml} | 5 +- .../{mysql-deployment.yaml => mysql.yaml} | 46 +++++-- .../authhub-web-secret.yaml => secrets.yaml} | 10 +- .../templates/web/authhub-web-config.yaml | 44 +++++++ .../templates/web/authhub-web-deployment.yaml | 51 -------- .../templates/web/authhub-web-ingress.yaml | 19 --- .../templates/web/authhub-web-service.yaml | 17 --- .../authhub/templates/web/authhub-web.yaml | 86 +++++++++++++ deploy/chart/authhub/values.yaml | 118 +++++++----------- .../databases/templates/mongo/mongo.yaml | 2 + .../databases/templates/pgsql/pgsql.yaml | 2 + 20 files changed, 333 insertions(+), 314 deletions(-) create mode 100644 deploy/chart/authhub/configs/backend/copy-config.yml delete mode 100644 deploy/chart/authhub/configs/web/authhub.nginx.conf rename deploy/chart/authhub/templates/backend/{authhub-backend-secret.yaml => authhub-backend-config.yaml} (66%) delete mode 100644 deploy/chart/authhub/templates/backend/authhub-backend-deployment.yaml delete mode 100644 deploy/chart/authhub/templates/backend/authhub-backend-service.yaml create mode 100644 deploy/chart/authhub/templates/backend/authhub-backend.yaml rename deploy/chart/authhub/templates/mysql/{mysql-secret.yaml => mysql-config.yaml} (54%) delete mode 100644 deploy/chart/authhub/templates/mysql/mysql-service.yaml rename deploy/chart/authhub/templates/mysql/{mysql-pvc.yaml => mysql-storage.yaml} (59%) rename deploy/chart/authhub/templates/mysql/{mysql-deployment.yaml => mysql.yaml} (59%) rename deploy/chart/authhub/templates/{web/authhub-web-secret.yaml => secrets.yaml} (33%) create mode 100644 deploy/chart/authhub/templates/web/authhub-web-config.yaml delete mode 100644 deploy/chart/authhub/templates/web/authhub-web-deployment.yaml delete mode 100644 deploy/chart/authhub/templates/web/authhub-web-ingress.yaml delete mode 100644 deploy/chart/authhub/templates/web/authhub-web-service.yaml create mode 100644 deploy/chart/authhub/templates/web/authhub-web.yaml diff --git a/deploy/chart/authhub/configs/backend/aops-config.yml b/deploy/chart/authhub/configs/backend/aops-config.yml index b43960d21..8ea4d5972 100644 --- a/deploy/chart/authhub/configs/backend/aops-config.yml +++ b/deploy/chart/authhub/configs/backend/aops-config.yml @@ -1,19 +1,19 @@ infrastructure: mysql: - host: mysql-db-{{ .Release.Name }}.{{ .Release.Namespace }}.svc.cluster.local + host: mysql-db.{{ .Release.Namespace }}.svc.cluster.local port: 3306 username: authhub pool_size: 100 pool_recycle: 7200 database: oauth2 - password: {{ .Values.authhub.mysql.password }} + password: ${mysql-password} redis: - host: redis-db-{{ .Values.globals.databases.app_name }}.{{ .Values.globals.databases.app_namespace }}.svc.cluster.local + host: redis-db.{{ .Release.Namespace }}.svc.cluster.local port: 6379 - password: {{ .Values.globals.databases.redis }} + password: ${redis-password} include: "/etc/aops/conf.d" -domain: {{ .Values.globals.domain }} +domain: {{ .Values.domain.authhub }} services: log: diff --git a/deploy/chart/authhub/configs/backend/copy-config.yml b/deploy/chart/authhub/configs/backend/copy-config.yml new file mode 100644 index 000000000..5df98a45a --- /dev/null +++ b/deploy/chart/authhub/configs/backend/copy-config.yml @@ -0,0 +1,7 @@ +copy: + - from: /config + to: /config-rw + mode: + uid: 1000 + gid: 1000 + mode: "0o750" \ No newline at end of file diff --git a/deploy/chart/authhub/configs/web/authhub.nginx.conf b/deploy/chart/authhub/configs/web/authhub.nginx.conf deleted file mode 100644 index 692df77b5..000000000 --- a/deploy/chart/authhub/configs/web/authhub.nginx.conf +++ /dev/null @@ -1,35 +0,0 @@ -server { - listen 8000; - server_name localhost; - - # gzip config - gzip on; - gzip_min_length 1k; - gzip_comp_level 6; - gzip_types text/plain text/css text/javascript application/json application/javascript application/x-javascript application/xml; - gzip_vary on; - gzip_disable "MSIE [1-6]\."; - - location / { - proxy_set_header X-Real-IP $remote_addr; - root /opt/authhub/web/dist; - index index.html; - try_files $uri $uri/ /index.html; - } - - location /authhub { - add_header Access-Control-Allow-Origin *; - add_header Access-Control-Allow-Methods 'GET, POST, DELETE, PUT, OPTIONS'; - alias /opt/authhub/web/dist; - index index.html; - try_files $uri $uri/ /index.html last; - } - - location /oauth2 { - proxy_pass http://authhub-backend-service-{{ .Release.Name }}.{{ .Release.Namespace }}.svc.cluster.local:11120; - proxy_set_header Host $host; - proxy_set_header X-Real-URL $request_uri; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Request-Header $http_request_header; - } -} diff --git a/deploy/chart/authhub/templates/backend/authhub-backend-secret.yaml b/deploy/chart/authhub/templates/backend/authhub-backend-config.yaml similarity index 66% rename from deploy/chart/authhub/templates/backend/authhub-backend-secret.yaml rename to deploy/chart/authhub/templates/backend/authhub-backend-config.yaml index def2b8a0d..bb91d55a6 100644 --- a/deploy/chart/authhub/templates/backend/authhub-backend-secret.yaml +++ b/deploy/chart/authhub/templates/backend/authhub-backend-config.yaml @@ -1,13 +1,14 @@ {{- if .Values.authhub.backend.enabled }} apiVersion: v1 -kind: Secret +kind: ConfigMap metadata: - name: authhub-backend-secret-{{ .Release.Name }} + name: authhub-backend-config namespace: {{ .Release.Namespace }} -type: Opaque -stringData: +data: aops-config.yml: |- {{ tpl (.Files.Get "configs/backend/aops-config.yml") . | indent 4 }} authhub.yml: |- {{ tpl (.Files.Get "configs/backend/authhub.yml") . | indent 4 }} + copy-config.yml: |- +{{ tpl (.Files.Get "configs/backend/copy-config.yml") . | indent 4 }} {{- end }} diff --git a/deploy/chart/authhub/templates/backend/authhub-backend-deployment.yaml b/deploy/chart/authhub/templates/backend/authhub-backend-deployment.yaml deleted file mode 100644 index 99d651754..000000000 --- a/deploy/chart/authhub/templates/backend/authhub-backend-deployment.yaml +++ /dev/null @@ -1,51 +0,0 @@ -{{- if .Values.authhub.backend.enabled }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: authhub-backend-deploy-{{ .Release.Name }} - namespace: {{ .Release.Namespace }} - labels: - app: authhub-backend-{{ .Release.Name }} -spec: - replicas: {{ .Values.globals.replicaCount }} - selector: - matchLabels: - app: authhub-backend-{{ .Release.Name }} - template: - metadata: - annotations: - checksum/secret: {{ include (print $.Template.BasePath "/backend/authhub-backend-secret.yaml") . | sha256sum }} - labels: - app: authhub-backend-{{ .Release.Name }} - spec: - automountServiceAccountToken: false - containers: - - name: authhub-backend - image: "{{if ne ( .Values.authhub.backend.image.registry | toString ) ""}}{{ .Values.authhub.backend.image.registry }}{{ else }}{{ .Values.globals.imageRegistry }}{{ end }}/{{ .Values.authhub.backend.image.name }}:{{ .Values.authhub.backend.image.tag | toString }}" - imagePullPolicy: {{ if ne ( .Values.authhub.backend.image.imagePullPolicy | toString ) "" }}{{ .Values.authhub.backend.image.imagePullPolicy }}{{ else }}{{ .Values.globals.imagePullPolicy }}{{ end }} - ports: - - containerPort: 11120 - protocol: TCP - volumeMounts: - - name: authhub-secret-volume - mountPath: /etc/aops - livenessProbe: - httpGet: - path: /oauth2/applications - port: 11120 - scheme: HTTP - failureThreshold: 5 - initialDelaySeconds: 60 - periodSeconds: 90 - securityContext: - readOnlyRootFilesystem: {{ .Values.authhub.backend.readOnly }} - volumes: - - name: authhub-secret-volume - secret: - secretName: authhub-backend-secret-{{ .Release.Name }} - items: - - key: aops-config.yml - path: aops-config.yml - - key: authhub.yml - path: conf.d/authhub.yml -{{- end }} \ No newline at end of file diff --git a/deploy/chart/authhub/templates/backend/authhub-backend-service.yaml b/deploy/chart/authhub/templates/backend/authhub-backend-service.yaml deleted file mode 100644 index 469c385e2..000000000 --- a/deploy/chart/authhub/templates/backend/authhub-backend-service.yaml +++ /dev/null @@ -1,17 +0,0 @@ -{{- if .Values.authhub.backend.enabled }} -apiVersion: v1 -kind: Service -metadata: - name: authhub-backend-service-{{ .Release.Name }} - namespace: {{ .Release.Namespace }} -spec: - type: {{ .Values.authhub.backend.service.type }} - selector: - app: authhub-backend-{{ .Release.Name }} - ports: - - port: 11120 - targetPort: 11120 - {{- if (and (eq .Values.authhub.backend.service.type "NodePort") .Values.authhub.backend.service.nodePort) }} - nodePort: {{ .Values.authhub.backend.service.nodePort }} - {{- end }} -{{- end }} \ No newline at end of file diff --git a/deploy/chart/authhub/templates/backend/authhub-backend.yaml b/deploy/chart/authhub/templates/backend/authhub-backend.yaml new file mode 100644 index 000000000..83a475cce --- /dev/null +++ b/deploy/chart/authhub/templates/backend/authhub-backend.yaml @@ -0,0 +1,91 @@ +{{- if .Values.authhub.backend.enabled }} +--- +apiVersion: v1 +kind: Service +metadata: + name: authhub-backend-service + namespace: {{ .Release.Namespace }} +spec: + type: {{ default "ClusterIP" .Values.authhub.backend.service.type }} + selector: + app: authhub-backend + ports: + - port: 11120 + targetPort: 11120 + nodePort: {{ default nil .Values.authhub.backend.service.nodePort }} + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: authhub-backend-deploy + namespace: {{ .Release.Namespace }} + labels: + app: authhub-backend +spec: + replicas: {{ default 1 .Values.globals.replicaCount }} + selector: + matchLabels: + app: authhub-backend + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/backend/authhub-backend-config.yaml") . | sha256sum }} + labels: + app: authhub-backend + spec: + automountServiceAccountToken: false + containers: + - name: authhub-backend + image: {{ default "hub.oepkgs.net/neocopilot/authhub:0.9.3-x86" .Values.authhub.backend.image }} + imagePullPolicy: {{ default "IfNotPresent" .Values.globals.imagePullPolicy }} + ports: + - containerPort: 11120 + protocol: TCP + volumeMounts: + - name: authhub-shared + mountPath: /etc/aops + livenessProbe: + httpGet: + path: /oauth2/applications + port: 11120 + scheme: HTTP + failureThreshold: 5 + initialDelaySeconds: 60 + periodSeconds: 90 + initContainers: + - name: authhub-backend-copy-secret + image: {{ default "hub.oepkgs.net/neocopilot/secret_inject:x86" .Values.authhub.secret_inject.image }} + imagePullPolicy: {{ default "IfNotPresent" .Values.globals.imagePullPolicy }} + volumeMounts: + - mountPath: /secrets/mysql-password + name: authhub-secret-vl + subPath: mysql-password + - mountPath: /secrets/redis-password + name: euler-copilot-database-vl + subPath: redis-password + - mountPath: /config/aops-config.yml + name: authhub-config + subPath: aops-config.yml + - mountPath: /config/conf.d/authhub.yml + name: authhub-config + subPath: authhub.yml + - mountPath: /config-rw + name: authhub-shared + - mountPath: /app/config.yaml + name: authhub-config + subPath: copy-config.yml + volumes: + - name: authhub-shared + emptyDir: + medium: Memory + - name: authhub-config + configMap: + name: authhub-backend-config + - name: authhub-secret-vl + secret: + secretName: authhub-secret + - name: euler-copilot-database-vl + secret: + secretName: euler-copilot-database +{{- end }} \ No newline at end of file diff --git a/deploy/chart/authhub/templates/mysql/mysql-secret.yaml b/deploy/chart/authhub/templates/mysql/mysql-config.yaml similarity index 54% rename from deploy/chart/authhub/templates/mysql/mysql-secret.yaml rename to deploy/chart/authhub/templates/mysql/mysql-config.yaml index d34cd531c..57e2dab92 100644 --- a/deploy/chart/authhub/templates/mysql/mysql-secret.yaml +++ b/deploy/chart/authhub/templates/mysql/mysql-config.yaml @@ -1,12 +1,10 @@ {{- if .Values.authhub.mysql.enabled }} apiVersion: v1 -kind: Secret +kind: ConfigMap metadata: - name: mysql-secret-{{ .Release.Name }} + name: mysql-config namespace: {{ .Release.Namespace }} -type: Opaque -stringData: - mysql-password: {{ .Values.authhub.mysql.password }} - init.sql: | +data: + init.sql: |- {{ tpl (.Files.Get "configs/mysql/init.sql") . | indent 4 }} {{- end }} \ No newline at end of file diff --git a/deploy/chart/authhub/templates/mysql/mysql-service.yaml b/deploy/chart/authhub/templates/mysql/mysql-service.yaml deleted file mode 100644 index 5d0cfd941..000000000 --- a/deploy/chart/authhub/templates/mysql/mysql-service.yaml +++ /dev/null @@ -1,17 +0,0 @@ -{{- if .Values.authhub.mysql.enabled }} -apiVersion: v1 -kind: Service -metadata: - name: mysql-db-{{ .Release.Name }} - namespace: {{ .Release.Namespace }} -spec: - type: {{ .Values.authhub.mysql.service.type }} - selector: - app: mysql-{{ .Release.Name }} - ports: - - port: 3306 - targetPort: 3306 - {{- if (and (eq .Values.authhub.mysql.service.type "NodePort") .Values.authhub.mysql.service.nodePort) }} - nodePort: {{ .Values.authhub.mysql.service.nodePort }} - {{- end }} -{{- end }} \ No newline at end of file diff --git a/deploy/chart/authhub/templates/mysql/mysql-pvc.yaml b/deploy/chart/authhub/templates/mysql/mysql-storage.yaml similarity index 59% rename from deploy/chart/authhub/templates/mysql/mysql-pvc.yaml rename to deploy/chart/authhub/templates/mysql/mysql-storage.yaml index 462a4a898..8197ac341 100644 --- a/deploy/chart/authhub/templates/mysql/mysql-pvc.yaml +++ b/deploy/chart/authhub/templates/mysql/mysql-storage.yaml @@ -2,14 +2,15 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: mysql-pvc-{{ .Release.Name }} + name: mysql-pvc namespace: {{ .Release.Namespace }} annotations: helm.sh/resource-policy: keep spec: + storageClassName: {{ default "local-path" .Values.globals.storageClassName }} accessModes: - ReadWriteOnce resources: requests: - storage: {{ .Values.authhub.mysql.persistentVolumeSize }} + storage: {{ default "10Gi" .Values.authhub.mysql.persistentVolumeSize }} {{- end }} \ No newline at end of file diff --git a/deploy/chart/authhub/templates/mysql/mysql-deployment.yaml b/deploy/chart/authhub/templates/mysql/mysql.yaml similarity index 59% rename from deploy/chart/authhub/templates/mysql/mysql-deployment.yaml rename to deploy/chart/authhub/templates/mysql/mysql.yaml index 511290e76..5e5db7622 100644 --- a/deploy/chart/authhub/templates/mysql/mysql-deployment.yaml +++ b/deploy/chart/authhub/templates/mysql/mysql.yaml @@ -1,28 +1,44 @@ {{- if .Values.authhub.mysql.enabled }} +--- +apiVersion: v1 +kind: Service +metadata: + name: mysql-db + namespace: {{ .Release.Namespace }} +spec: + type: {{ default "ClusterIP" .Values.authhub.mysql.service.type }} + selector: + app: mysql + ports: + - port: 3306 + targetPort: 3306 + nodePort: {{ default nil .Values.authhub.mysql.service.nodePort }} + +--- apiVersion: apps/v1 kind: Deployment metadata: - name: mysql-deploy-{{ .Release.Name }} + name: mysql-deploy namespace: {{ .Release.Namespace }} labels: - app: mysql-{{ .Release.Name }} + app: mysql spec: - replicas: {{ .Values.globals.replicaCount }} + replicas: {{ default 1 .Values.globals.replicaCount }} selector: matchLabels: - app: mysql-{{ .Release.Name }} + app: mysql template: metadata: annotations: - checksum/secret: {{ include (print $.Template.BasePath "/mysql/mysql-secret.yaml") . | sha256sum }} + checksum/config: {{ include (print $.Template.BasePath "/mysql/mysql-config.yaml") . | sha256sum }} labels: - app: mysql-{{ .Release.Name }} + app: mysql spec: automountServiceAccountToken: false containers: - name: mysql - image: "{{ if ne (.Values.authhub.mysql.image.registry | toString ) "" }}{{ .Values.authhub.mysql.image.registry }}{{ else }}{{ .Values.globals.imageRegistry }}{{ end }}/{{ .Values.authhub.mysql.image.name }}:{{ .Values.authhub.mysql.image.tag | toString }}" - imagePullPolicy: {{ if ne (.Values.authhub.mysql.image.imagePullPolicy | toString) "" }}{{ .Values.authhub.mysql.image.imagePullPolicy }}{{ else }}{{ .Values.globals.imagePullPolicy }}{{ end }} + image: {{ default "hub.oepkgs.net/neocopilot/mysql:8-x86" .Values.authhub.mysql.image }} + imagePullPolicy: {{ default "IfNotPresent" .Values.globals.imagePullPolicy }} args: - "--character-set-server=utf8mb4" - "--collation-server=utf8mb4_unicode_ci" @@ -48,7 +64,7 @@ spec: - name: MYSQL_PASSWORD valueFrom: secretKeyRef: - name: mysql-secret-{{ .Release.Name }} + name: authhub-secret key: mysql-password volumeMounts: - mountPath: /var/lib/mysql @@ -57,13 +73,17 @@ spec: name: mysql-init subPath: init.sql resources: - {{- toYaml .Values.authhub.mysql.resources | nindent 12 }} + requests: + cpu: 0.1 + memory: 256Mi + limits: + {{- toYaml .Values.authhub.mysql.resourceLimits | nindent 14 }} restartPolicy: Always volumes: - name: mysql-data persistentVolumeClaim: - claimName: mysql-pvc-{{ .Release.Name }} + claimName: mysql-pvc - name: mysql-init - secret: - secretName: mysql-secret-{{ .Release.Name }} + configMap: + name: mysql-config {{- end }} diff --git a/deploy/chart/authhub/templates/web/authhub-web-secret.yaml b/deploy/chart/authhub/templates/secrets.yaml similarity index 33% rename from deploy/chart/authhub/templates/web/authhub-web-secret.yaml rename to deploy/chart/authhub/templates/secrets.yaml index b2447d5b0..aca33aa25 100644 --- a/deploy/chart/authhub/templates/web/authhub-web-secret.yaml +++ b/deploy/chart/authhub/templates/secrets.yaml @@ -1,11 +1,13 @@ -{{- if .Values.authhub.web.enabled }} +{{- $authhubSecret := (lookup "v1" "Secret" .Release.Namespace "authhub-secret") }} +{{- if not $authhubSecret}} apiVersion: v1 kind: Secret metadata: - name: authhub-web-secret-{{ .Release.Name }} + name: authhub-secret namespace: {{ .Release.Namespace }} + annotations: + helm.sh/resource-policy: keep type: Opaque stringData: - authhub.nginx.conf: |- -{{ tpl (.Files.Get "configs/web/authhub.nginx.conf") . | indent 4 }} + mysql-password: {{ randAlphaNum 20 }} {{- end }} \ No newline at end of file diff --git a/deploy/chart/authhub/templates/web/authhub-web-config.yaml b/deploy/chart/authhub/templates/web/authhub-web-config.yaml new file mode 100644 index 000000000..0d1cdf153 --- /dev/null +++ b/deploy/chart/authhub/templates/web/authhub-web-config.yaml @@ -0,0 +1,44 @@ +{{- if .Values.authhub.web.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: authhub-web-config + namespace: {{ .Release.Namespace }} +data: + authhub.nginx.conf: |- + server { + listen 8000; + server_name localhost; + + # gzip config + gzip on; + gzip_min_length 1k; + gzip_comp_level 6; + gzip_types text/plain text/css text/javascript application/json application/javascript application/x-javascript application/xml; + gzip_vary on; + gzip_disable "MSIE [1-6]\."; + + location / { + proxy_set_header X-Real-IP $remote_addr; + root /opt/authhub/web/dist; + index index.html; + try_files $uri $uri/ /index.html; + } + + location /authhub { + add_header Access-Control-Allow-Origin *; + add_header Access-Control-Allow-Methods 'GET, POST, DELETE, PUT, OPTIONS'; + alias /opt/authhub/web/dist; + index index.html; + try_files $uri $uri/ /index.html last; + } + + location /oauth2 { + proxy_pass http://authhub-backend-service.{{ .Release.Namespace }}.svc.cluster.local:11120; + proxy_set_header Host $host; + proxy_set_header X-Real-URL $request_uri; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Request-Header $http_request_header; + } + } +{{- end }} \ No newline at end of file diff --git a/deploy/chart/authhub/templates/web/authhub-web-deployment.yaml b/deploy/chart/authhub/templates/web/authhub-web-deployment.yaml deleted file mode 100644 index 220249235..000000000 --- a/deploy/chart/authhub/templates/web/authhub-web-deployment.yaml +++ /dev/null @@ -1,51 +0,0 @@ -{{- if .Values.authhub.web.enabled }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: authhub-web-deploy-{{ .Release.Name }} - namespace: {{ .Release.Namespace }} - labels: - app: authhub-web-{{ .Release.Name }} -spec: - replicas: {{ .Values.globals.replicaCount }} - selector: - matchLabels: - app: authhub-web-{{ .Release.Name }} - template: - metadata: - annotations: - checksum/secret: {{ include (print $.Template.BasePath "/web/authhub-web-secret.yaml") . | sha256sum }} - labels: - app: authhub-web-{{ .Release.Name }} - spec: - automountServiceAccountToken: false - containers: - - name: authhub-web - image: "{{if ne ( .Values.authhub.web.image.registry | toString ) ""}}{{ .Values.authhub.web.image.registry }}{{ else }}{{ .Values.globals.imageRegistry }}{{ end }}/{{ .Values.authhub.web.image.name }}:{{ .Values.authhub.web.image.tag | toString }}" - imagePullPolicy: {{ if ne ( .Values.authhub.web.image.imagePullPolicy | toString ) "" }}{{ .Values.authhub.web.image.imagePullPolicy }}{{ else }}{{ .Values.globals.imagePullPolicy }}{{ end }} - ports: - - containerPort: 8000 - protocol: TCP - livenessProbe: - httpGet: - path: / - port: 8000 - scheme: HTTP - failureThreshold: 5 - initialDelaySeconds: 60 - periodSeconds: 90 - volumeMounts: - - name: authhub-web-secret-volume - mountPath: /etc/nginx/conf.d - securityContext: - readOnlyRootFilesystem: {{ .Values.authhub.web.readOnly }} - resources: - {{- toYaml .Values.authhub.web.resources | nindent 12 }} - volumes: - - name: authhub-web-secret-volume - secret: - secretName: authhub-web-secret-{{ .Release.Name }} - items: - - key: authhub.nginx.conf - path: authhub.nginx.conf -{{- end }} diff --git a/deploy/chart/authhub/templates/web/authhub-web-ingress.yaml b/deploy/chart/authhub/templates/web/authhub-web-ingress.yaml deleted file mode 100644 index 4d08eb0f0..000000000 --- a/deploy/chart/authhub/templates/web/authhub-web-ingress.yaml +++ /dev/null @@ -1,19 +0,0 @@ -{{- if .Values.authhub.web.ingress.enabled }} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: authhub-web-ingress-{{ .Release.Name }} - namespace: {{ .Release.Namespace }} -spec: - rules: - - host: {{ .Values.globals.domain }} - http: - paths: - - path: {{ .Values.authhub.web.ingress.prefix }} - pathType: Prefix - backend: - service: - name: authhub-web-service-{{ .Release.Name }} - port: - number: 8000 -{{- end }} \ No newline at end of file diff --git a/deploy/chart/authhub/templates/web/authhub-web-service.yaml b/deploy/chart/authhub/templates/web/authhub-web-service.yaml deleted file mode 100644 index 774f2017a..000000000 --- a/deploy/chart/authhub/templates/web/authhub-web-service.yaml +++ /dev/null @@ -1,17 +0,0 @@ -{{- if .Values.authhub.web.enabled }} -apiVersion: v1 -kind: Service -metadata: - name: authhub-web-service-{{ .Release.Name }} - namespace: {{ .Release.Namespace }} -spec: - type: {{ .Values.authhub.web.service.type }} - selector: - app: authhub-web-{{ .Release.Name }} - ports: - - port: 8000 - targetPort: 8000 - {{- if (and (eq .Values.authhub.web.service.type "NodePort") .Values.authhub.web.service.nodePort) }} - nodePort: {{ .Values.authhub.web.service.nodePort }} - {{- end }} -{{- end }} \ No newline at end of file diff --git a/deploy/chart/authhub/templates/web/authhub-web.yaml b/deploy/chart/authhub/templates/web/authhub-web.yaml new file mode 100644 index 000000000..86528dd93 --- /dev/null +++ b/deploy/chart/authhub/templates/web/authhub-web.yaml @@ -0,0 +1,86 @@ +{{- if .Values.authhub.web.enabled }} +--- +apiVersion: v1 +kind: Service +metadata: + name: authhub-web-service + namespace: {{ .Release.Namespace }} +spec: + type: {{ default "ClusterIP" .Values.authhub.web.service.type }} + selector: + app: authhub-web + ports: + - port: 8000 + targetPort: 8000 + nodePort: {{ default nil .Values.authhub.web.service.nodePort }} + +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: authhub-web-ingress + namespace: {{ .Release.Namespace }} +spec: + rules: + - host: {{ default "authhub.eulercopilot.local" .Values.domain.authhub }} + http: + paths: + - path: {{ default "/" .Values.authhub.web.ingress.prefix }} + pathType: Prefix + backend: + service: + name: authhub-web-service + port: + number: 8000 + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: authhub-web-deploy + namespace: {{ .Release.Namespace }} + labels: + app: authhub-web +spec: + replicas: {{ default 1 .Values.globals.replicaCount }} + selector: + matchLabels: + app: authhub-web + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/web/authhub-web-config.yaml") . | sha256sum }} + labels: + app: authhub-web + spec: + automountServiceAccountToken: false + containers: + - name: authhub-web + image: {{ default "hub.oepkgs.net/neocopilot/authhub-web:0.9.3-x86" .Values.authhub.web.image }} + imagePullPolicy: {{ default "IfNotPresent" .Values.globals.imagePullPolicy }} + ports: + - containerPort: 8000 + protocol: TCP + livenessProbe: + httpGet: + path: / + port: 8000 + scheme: HTTP + failureThreshold: 5 + initialDelaySeconds: 60 + periodSeconds: 90 + volumeMounts: + - name: web-config + mountPath: /etc/nginx/conf.d/authhub.nginx.conf + subPath: authhub.nginx.conf + resources: + requests: + cpu: 0.05 + memory: 64Mi + limits: + {{- toYaml .Values.authhub.web.resourceLimits | nindent 14 }} + volumes: + - name: web-config + configMap: + name: authhub-web-config +{{- end }} diff --git a/deploy/chart/authhub/values.yaml b/deploy/chart/authhub/values.yaml index 15684c009..4e98e533d 100644 --- a/deploy/chart/authhub/values.yaml +++ b/deploy/chart/authhub/values.yaml @@ -1,101 +1,73 @@ # 全局设置 globals: - # [必填] 镜像仓库 - imageRegistry: "hub.oepkgs.net/neocopilot" - # [必填] 镜像拉取策略 - imagePullPolicy: IfNotPresent - # [必填] AuthHub部署域名 - # 需要修改为AuthHub域名。单机部署时，服务基于Host进行区分，无法使用IP地址 - domain: "" - # [必填] 副本数 - replicaCount: 1 - # [必填] databases chart的信息 - databases: - # [必填] helm安装时的release name - app_name: "databases" - # [必填] helm安装时的namespace - app_namespace: "euler-copilot" - # [必填] redis密码 - redis: "admin123" + # 镜像拉取策略；默认为IfNotPresent + imagePullPolicy: + # 副本数，默认为1 + replicaCount: + # 存储类名称；默认为local-path + storageClassName: + +storage: + # MySQL持久化存储大小，默认为10Gi + mysql: + +domain: + # AuthHub域名，默认为authhub.eulercopilot.local。单机部署时，服务基于Host进行区分，无法使用IP地址 + authhub: # 部署AuthHub本地鉴权服务 authhub: + # 配置文件工具 + secret_inject: + # 镜像设置；默认为hub.oepkgs.net/neocopilot/secret_inject:x86 + # 镜像标签：["x86", "arm"] + image: hub.oepkgs.net/neocopilot/secret_inject:dev + web: # [必填] 是否部署AuthHub前端服务 enabled: true - # 镜像设置 + # 镜像设置；默认为hub.oepkgs.net/neocopilot/authhub-web:0.9.3-x86 + # 镜像标签：["0.9.3-x86", "0.9.3-arm"] image: - # 镜像仓库。留空则使用全局设置。 - registry: "" - # [必填] 镜像名 - name: authhub-web - # [必填] 镜像Tag, 为0.9.3-x86或0.9.3-arm - tag: "0.9.3-x86" - # 拉取策略。留空则使用全局设置。 - imagePullPolicy: "" - # [必填] 容器根目录只读 - readOnly: false # 性能限制设置 - resources: {} + resourceLimits: {} # Service设置 service: - # [必填] Service类型，ClusterIP或NodePort - type: ClusterIP - # 当类型为nodePort时，填写主机的端口号 - nodePort: "" + # Service类型，例如NodePort + type: + # 当类型为NodePort时，填写主机的端口号 + nodePort: # Ingress设置 ingress: - # [必填] 是否启用Ingress - enabled: true - # [必填] URI前缀 - prefix: / + # Ingress前缀，默认为/ + prefix: + backend: # [必填] 是否部署AuthHub后端服务 enabled: true - # 镜像设置 + # 镜像设置；默认为hub.oepkgs.net/neocopilot/authhub:0.9.3-x86 + # 镜像标签：["0.9.3-x86", "0.9.3-arm"] image: - # 镜像仓库。留空则使用全局设置。 - registry: "" - # [必填] 镜像名 - name: authhub - # 镜像Tag,为0.9.3-x86或0.9.3-arm - tag: "0.9.3-x86" - # 拉取策略。留空则使用全局设置。 - imagePullPolicy: "" - # [必填] 容器根目录只读 - readOnly: false # 性能限制设置 - resources: {} + resourceLimits: {} # Service设置 service: - # [必填] Service类型，ClusterIP或NodePort - type: ClusterIP - # 当类型为nodePort时，填写主机的端口号 - nodePort: "" + # Service类型，例如NodePort + type: + # 当类型为NodePort时，填写主机的端口号 + nodePort: + mysql: # [必填] 是否启用MySQL enabled: true - # 镜像设置 + # 镜像设置；默认为hub.oepkgs.net/neocopilot/mysql:8-x86 + # 镜像标签：["8-x86", "8-arm"] image: - # 镜像仓库。留空则使用全局设置。 - registry: "" - # [必填] 镜像名 - name: mysql - # [必填] 镜像Tag,为8-x86或8-arm - tag: "8-x86" - # 拉取策略。留空则使用全局设置。 - imagePullPolicy: "" - # [必填] 容器根目录只读 - readOnly: false # 性能限制设置 - resources: {} + resourceLimits: {} # Service设置 service: - # [必填] Service类型，ClusterIP或NodePort - type: ClusterIP - # 当类型为nodePort时，填写主机的端口号 - nodePort: "" - # [必填] 密码 - password: "admin123" - # [必填] 持久化存储大小 - persistentVolumeSize: 10Gi + # Service类型，例如NodePort + type: + # 当类型为NodePort时，填写主机的端口号 + nodePort: diff --git a/deploy/chart/databases/templates/mongo/mongo.yaml b/deploy/chart/databases/templates/mongo/mongo.yaml index 46dbdd8e2..1d058a915 100644 --- a/deploy/chart/databases/templates/mongo/mongo.yaml +++ b/deploy/chart/databases/templates/mongo/mongo.yaml @@ -29,6 +29,8 @@ spec: app: mongo template: metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/mongo/mongo-config.yaml") . | sha256sum }} labels: app: mongo spec: diff --git a/deploy/chart/databases/templates/pgsql/pgsql.yaml b/deploy/chart/databases/templates/pgsql/pgsql.yaml index 3ade7ee1b..9d5e06c3c 100644 --- a/deploy/chart/databases/templates/pgsql/pgsql.yaml +++ b/deploy/chart/databases/templates/pgsql/pgsql.yaml @@ -29,6 +29,8 @@ spec: app: pgsql template: metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/pgsql/pgsql-config.yaml") . | sha256sum }} labels: app: pgsql spec: -- Gitee From ea57cf60cfa39e75f12104ca27d10b47d5be16b7 Mon Sep 17 00:00:00 2001 From: z30057876 Date: Tue, 11 Feb 2025 20:25:41 +0800 Subject: [PATCH 3/3] =?UTF-8?q?=E5=BC=83=E7=94=A8vectorize=EF=BC=8C?= =?UTF-8?q?=E9=80=82=E9=85=8Dopenai=20embedding=20api?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/common/config.py | 6 ++++-- apps/scheduler/vector.py | 17 ++++++++++++++--- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/apps/common/config.py b/apps/common/config.py index 8f487357a..12da81e2a 100644 --- a/apps/common/config.py +++ b/apps/common/config.py @@ -37,8 +37,10 @@ class ConfigModel(BaseModel): SESSION_TTL: int = Field(description="用户需要刷新Token的间隔(min)", default=30) # Logging LOG: str = Field(description="日志记录模式") - # Vectorize - VECTORIZE_HOST: str = Field(description="Vectorize服务域名") + # Embedding + EMBEDDING_URL: str = Field(description="Embedding模型地址") + EMBEDDING_KEY: str = Field(description="Embedding模型API Key") + EMBEDDING_MODEL: str = Field(description="Embedding模型名称") # RAG RAG_HOST: str = Field(description="RAG服务域名") # FastAPI diff --git a/apps/scheduler/vector.py b/apps/scheduler/vector.py index f789a3d9e..5cf2c7460 100644 --- a/apps/scheduler/vector.py +++ b/apps/scheduler/vector.py @@ -27,15 +27,26 @@ def _get_embedding(text: list[str]) -> list[np.ndarray]: :param text: 待向量化文本（多条文本组成List） :return: 文本对应的向量（顺序与text一致，也为List） """ - api = config["VECTORIZE_HOST"].rstrip("/") + "/embedding" + api = config["EMBEDDING_URL"].rstrip("/") + "/embedding" + + headers = { + "Authorization": f"Bearer {config['EMBEDDING_KEY']}", + } + data = { + "encoding_format": "float", + "input": text, + "model": config["EMBEDDING_MODEL"], + } + response = requests.post( api, - json={"texts": text}, + json=data, + headers=headers, verify=False, # noqa: S501 timeout=30, ) - return [np.array(vec) for vec in response.json()] + return [np.array(item["embedding"]) for item in response.json()["data"]] # 模块内部类，不应在模块外部使用 -- Gitee