From 6dfd828fe50845367886176c9fc43ddec63c91d8 Mon Sep 17 00:00:00 2001 From: Toyas Date: Sat, 18 Apr 2026 18:22:35 +0800 Subject: [PATCH] chore(config): Unify deployment orchestration and monitoring integration --- .gitignore | 3 +- backend/.env.example | 17 + backend/config/.env.node-01 | 14 +- backend/config/.env.node-02 | 10 +- backend/config/.env.node-03 | 10 +- backend/config/redis/sentinel-single.conf | 10 + backend/deploy-ha.sh | 2 +- backend/docker-compose-node-01.yml | 87 ++ backend/docker-compose-node-02.yml | 50 + backend/docker-compose-node-03.yml | 50 + backend/docker-compose.yml | 389 ++++++-- backend/monitoring/README.md | 40 + .../grafana/dashboards/ha-ovp-overview.json | 887 ++++++++++++++++++ .../provisioning/dashboards/dashboard.yml | 11 + .../provisioning/datasources/datasource.yml | 10 + .../monitoring/prometheus/prometheus.ha.yml | 207 ++++ .../prometheus/prometheus.local.yml | 86 ++ backend/pom.xml | 3 +- backend/video-admin/pom.xml | 7 +- backend/video-auth/pom.xml | 7 +- backend/video-comment/pom.xml | 7 +- backend/video-content/pom.xml | 7 +- backend/video-gateway/pom.xml | 12 +- .../gateway/filter/AuthGlobalFilter.java | 34 +- .../src/main/resources/application-prod.yml | 12 +- .../src/main/resources/application.yml | 1 + backend/video-recommend/pom.xml | 7 +- backend/video-user/pom.xml | 7 +- docs/DEBIAN12-DEPLOYMENT-GUIDE.md | 7 +- docs/HA-DEPLOYMENT-GUIDE.md | 339 ------- 30 files changed, 1901 insertions(+), 432 deletions(-) create mode 100644 backend/.env.example create mode 100644 backend/config/redis/sentinel-single.conf create mode 100644 backend/monitoring/README.md create mode 100644 backend/monitoring/grafana/dashboards/ha-ovp-overview.json create mode 100644 backend/monitoring/grafana/provisioning/dashboards/dashboard.yml create mode 100644 backend/monitoring/grafana/provisioning/datasources/datasource.yml create mode 100644 backend/monitoring/prometheus/prometheus.ha.yml create mode 100644 backend/monitoring/prometheus/prometheus.local.yml delete mode 100644 docs/HA-DEPLOYMENT-GUIDE.md diff --git a/.gitignore b/.gitignore index cabb779..c97cd9d 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ local/ .output/ # 环境变量 (保护本地密码和密钥不被泄露) +.env .env.local .env.*.local @@ -80,4 +81,4 @@ pnpm-debug.log* Thumbs.db ehthumbs.db Desktop.ini -$RECYCLE.BIN/ \ No newline at end of file +$RECYCLE.BIN/ diff --git a/backend/.env.example b/backend/.env.example new file mode 100644 index 0000000..72b3580 --- /dev/null +++ b/backend/.env.example @@ -0,0 +1,17 @@ +MYSQL_ROOT_PASSWORD=replace-with-strong-mysql-root-password +MYSQL_APP_USER=video +MYSQL_APP_PASSWORD=replace-with-strong-mysql-app-password +REDIS_PASSWORD=replace-with-strong-redis-password +MINIO_ACCESS_KEY=replace-with-minio-access-key +MINIO_SECRET_KEY=replace-with-minio-secret-key +MINIO_BUCKET=video-platform +USER_JWT_SECRET=replace-with-strong-user-jwt-secret +ADMIN_JWT_SECRET=replace-with-strong-admin-jwt-secret +SERVICE_AUTH_TOKEN=replace-with-strong-internal-service-token +ADMIN_INIT_PASSWORD=replace-with-initial-admin-password +JWT_EXPIRATION=86400 +NACOS_USERNAME=nacos +NACOS_PASSWORD=replace-with-nacos-password +NACOS_AUTH_TOKEN=replace-with-nacos-auth-token +GRAFANA_ADMIN_USER=admin +GRAFANA_ADMIN_PASSWORD=replace-with-grafana-password diff --git a/backend/config/.env.node-01 b/backend/config/.env.node-01 index 272a576..99ea9fe 100644 --- a/backend/config/.env.node-01 +++ b/backend/config/.env.node-01 @@ -16,15 +16,15 @@ NAMESRV_ADDR=192.168.1.101:9876;192.168.1.102:9876;192.168.1.103:9876 MYSQL_SERVER_ID=1 MYSQL_HOST=mysql MYSQL_PORT=3306 -MYSQL_ROOT_PASSWORD=root123456 +MYSQL_ROOT_PASSWORD=replace-with-strong-mysql-root-password MYSQL_DATABASE=video_platform MYSQL_USER=video -MYSQL_PASSWORD=video123 +MYSQL_PASSWORD=replace-with-strong-mysql-app-password # Redis配置 REDIS_HOST=redis REDIS_PORT=6379 -REDIS_PASSWORD=redis_HA_Cluster_2024 +REDIS_PASSWORD=replace-with-strong-redis-password REDIS_MASTER_HOST=192.168.1.101 REDIS_MASTER_PORT=6379 @@ -42,8 +42,8 @@ SENTINEL_PORT=8080 MINIO_HOST=minio MINIO_PORT=9000 MINIO_EXTERNAL_PORT=9100 -MINIO_ACCESS_KEY=minioadmin -MINIO_SECRET_KEY=minioadmin123 +MINIO_ACCESS_KEY=replace-with-minio-access-key +MINIO_SECRET_KEY=replace-with-minio-secret-key MINIO_BUCKET=video-platform # RocketMQ配置 @@ -53,5 +53,9 @@ ROCKETMQ_PORT=9876 # Java配置 JAVA_OPTS=-Xms256m -Xmx512m -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=256m +# Grafana配置 +GRAFANA_ADMIN_USER=admin +GRAFANA_ADMIN_PASSWORD=change-this-grafana-password + # 本节点部署的服务 DEPLOY_SERVICES=mysql,redis,redis-sentinel,nacos,rmqnamesrv,minio,sentinel,video-gateway,video-auth,video-user,video-content,video-comment,video-recommend,video-admin,nginx diff --git a/backend/config/.env.node-02 b/backend/config/.env.node-02 index 3bab818..5a6d936 100644 --- a/backend/config/.env.node-02 +++ b/backend/config/.env.node-02 @@ -16,15 +16,15 @@ NAMESRV_ADDR=192.168.1.101:9876;192.168.1.102:9876;192.168.1.103:9876 MYSQL_SERVER_ID=2 MYSQL_HOST=mysql MYSQL_PORT=3306 -MYSQL_ROOT_PASSWORD=root123456 +MYSQL_ROOT_PASSWORD=replace-with-strong-mysql-root-password MYSQL_DATABASE=video_platform MYSQL_USER=video -MYSQL_PASSWORD=video123 +MYSQL_PASSWORD=replace-with-strong-mysql-app-password # Redis配置 REDIS_HOST=redis REDIS_PORT=6379 -REDIS_PASSWORD=redis_HA_Cluster_2024 +REDIS_PASSWORD=replace-with-strong-redis-password REDIS_MASTER_HOST=192.168.1.101 REDIS_MASTER_PORT=6379 @@ -42,8 +42,8 @@ SENTINEL_PORT=8080 MINIO_HOST=minio MINIO_PORT=9000 MINIO_EXTERNAL_PORT=9100 -MINIO_ACCESS_KEY=minioadmin -MINIO_SECRET_KEY=minioadmin123 +MINIO_ACCESS_KEY=replace-with-minio-access-key +MINIO_SECRET_KEY=replace-with-minio-secret-key MINIO_BUCKET=video-platform # RocketMQ配置 (Broker-A) diff --git a/backend/config/.env.node-03 b/backend/config/.env.node-03 index c27b2ae..434307b 100644 --- a/backend/config/.env.node-03 +++ b/backend/config/.env.node-03 @@ -16,15 +16,15 @@ NAMESRV_ADDR=192.168.1.101:9876;192.168.1.102:9876;192.168.1.103:9876 MYSQL_SERVER_ID=3 MYSQL_HOST=mysql MYSQL_PORT=3306 -MYSQL_ROOT_PASSWORD=root123456 +MYSQL_ROOT_PASSWORD=replace-with-strong-mysql-root-password MYSQL_DATABASE=video_platform MYSQL_USER=video -MYSQL_PASSWORD=video123 +MYSQL_PASSWORD=replace-with-strong-mysql-app-password # Redis配置 REDIS_HOST=redis REDIS_PORT=6379 -REDIS_PASSWORD=redis_HA_Cluster_2024 +REDIS_PASSWORD=replace-with-strong-redis-password REDIS_MASTER_HOST=192.168.1.101 REDIS_MASTER_PORT=6379 @@ -41,8 +41,8 @@ SENTINEL_PORT=8080 # MinIO配置 MINIO_HOST=minio MINIO_PORT=9000 -MINIO_ACCESS_KEY=minioadmin -MINIO_SECRET_KEY=minioadmin123 +MINIO_ACCESS_KEY=replace-with-minio-access-key +MINIO_SECRET_KEY=replace-with-minio-secret-key MINIO_BUCKET=video-platform # MinIO分布式节点 MINIO_ENDPOINTS=http://192.168.1.101:9000,http://192.168.1.102:9000,http://192.168.1.103:9000 diff --git a/backend/config/redis/sentinel-single.conf b/backend/config/redis/sentinel-single.conf new file mode 100644 index 0000000..0f68049 --- /dev/null +++ b/backend/config/redis/sentinel-single.conf @@ -0,0 +1,10 @@ +port 26379 +dir /data +logfile "" + +sentinel resolve-hostnames yes +sentinel monitor mymaster redis 6379 1 +sentinel auth-pass mymaster __REDIS_PASSWORD__ +sentinel down-after-milliseconds mymaster 30000 +sentinel failover-timeout mymaster 180000 +sentinel parallel-syncs mymaster 1 diff --git a/backend/deploy-ha.sh b/backend/deploy-ha.sh index b28a81e..31f7445 100644 --- a/backend/deploy-ha.sh +++ b/backend/deploy-ha.sh @@ -276,7 +276,7 @@ init_minio() { log_info "创建存储桶..." docker run --rm --network ha-network minio/mc < /data/sentinel.conf && redis-sentinel /data/sentinel.conf" + depends_on: + redis: + condition: service_healthy networks: - video-network @@ -39,16 +66,33 @@ services: restart: always environment: - MODE=standalone + - SPRING_DATASOURCE_PLATFORM=mysql + - MYSQL_SERVICE_HOST=mysql + - MYSQL_SERVICE_PORT=3306 + - MYSQL_SERVICE_DB_NAME=nacos_config + - MYSQL_SERVICE_USER=root + - MYSQL_SERVICE_PASSWORD=${MYSQL_ROOT_PASSWORD} + - MYSQL_SERVICE_DB_PARAM=characterEncoding=utf8&connectTimeout=10000&socketTimeout=30000&autoReconnect=true&useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true - JVM_XMS=512m - JVM_XMX=512m - # 添加Nacos参数，确保正确处理服务注册 - - NACOS_AUTH_ENABLE=false + - NACOS_AUTH_ENABLE=true + - NACOS_AUTH_TOKEN=${NACOS_AUTH_TOKEN} + - NACOS_AUTH_IDENTITY_KEY=serverIdentity + - NACOS_AUTH_IDENTITY_VALUE=security - PREFER_HOST_MODE=hostname - NACOS_SERVER_IP=${HOST_IP:-127.0.0.1} ports: - "8848:8848" - "9848:9848" - "9849:9849" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8848/nacos/v1/console/health/liveness"] + interval: 10s + timeout: 5s + retries: 10 + depends_on: + mysql: + condition: service_healthy networks: - video-network @@ -59,22 +103,35 @@ services: ports: - "9876:9876" command: sh mqnamesrv + healthcheck: + test: ["CMD-SHELL", "pgrep -f 'NamesrvStartup' || exit 1"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s networks: - video-network rmqbroker: image: apache/rocketmq:5.1.4 + user: root container_name: rmqbroker restart: always ports: - "10909:10909" - "10911:10911" + - "10912:10912" environment: - NAMESRV_ADDR=rmqnamesrv:9876 - JAVA_OPTS=-Duser.home=/opt + volumes: + - ./config/rocketmq/broker-a.conf:/home/rocketmq/rocketmq-5.1.4/conf/broker.conf:ro + - rmqbroker_logs:/home/rocketmq/logs + - rmqbroker_store:/home/rocketmq/store command: sh mqbroker -c /home/rocketmq/rocketmq-5.1.4/conf/broker.conf depends_on: - - rmqnamesrv + rmqnamesrv: + condition: service_healthy networks: - video-network @@ -83,18 +140,26 @@ services: container_name: minio restart: always ports: - - "9002:9000" - - "9001:9001" + - "9100:9000" + - "9101:9001" environment: - - MINIO_ROOT_USER=minioadmin - - MINIO_ROOT_PASSWORD=minioadmin + - MINIO_ROOT_USER=${MINIO_ACCESS_KEY} + - MINIO_ROOT_PASSWORD=${MINIO_SECRET_KEY} volumes: - minio_data:/data command: server /data --console-address ":9001" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 10s + timeout: 5s + retries: 5 networks: - video-network sentinel: + build: + context: ./sentinel-dashboard + dockerfile: Dockerfile image: sentinel-dashboard:1.8.6 container_name: sentinel restart: always @@ -103,6 +168,85 @@ services: networks: - video-network + prometheus: + image: prom/prometheus:v2.54.1 + container_name: prometheus + restart: always + ports: + - "9090:9090" + volumes: + - ./monitoring/prometheus/prometheus.local.yml:/etc/prometheus/prometheus.yml:ro + - prometheus_data:/prometheus + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=15d + networks: + - video-network + + grafana: + image: grafana/grafana:11.2.0 + container_name: grafana + restart: always + ports: + - "3000:3000" + environment: + - GF_SECURITY_ADMIN_USER=${GRAFANA_ADMIN_USER} + - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD} + - GF_USERS_ALLOW_SIGN_UP=false + volumes: + - grafana_data:/var/lib/grafana + - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro + - ./monitoring/grafana/dashboards:/var/lib/grafana/dashboards:ro + depends_on: + - prometheus + networks: + - video-network + + mysqld-exporter: + image: prom/mysqld-exporter:v0.16.0 + container_name: mysqld-exporter + restart: always + environment: + - MYSQLD_EXPORTER_PASSWORD=${MYSQL_ROOT_PASSWORD} + command: + - --mysqld.address=mysql:3306 + - --mysqld.username=root + depends_on: + mysql: + condition: service_healthy + networks: + - video-network + + redis-exporter: + image: oliver006/redis_exporter:v1.62.0 + container_name: redis-exporter + restart: always + environment: + - REDIS_ADDR=redis://redis:6379 + - REDIS_PASSWORD=${REDIS_PASSWORD} + depends_on: + redis: + condition: service_healthy + networks: + - video-network + + node-exporter: + image: prom/node-exporter:v1.8.2 + container_name: node-exporter + restart: always + command: + - --path.procfs=/host/proc + - --path.sysfs=/host/sys + - --path.rootfs=/rootfs + - --collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($|/) + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + networks: + - video-network + # 微服务 video-gateway: build: @@ -113,16 +257,30 @@ services: ports: - "9000:9000" environment: - - SPRING_PROFILES_ACTIVE=prod + - SPRING_PROFILES_ACTIVE=ha - NACOS_HOST=nacos - NACOS_PORT=8848 + - NACOS_USERNAME=${NACOS_USERNAME} + - NACOS_PASSWORD=${NACOS_PASSWORD} + - SPRING_CLOUD_NACOS_DISCOVERY_USERNAME=${NACOS_USERNAME} + - SPRING_CLOUD_NACOS_DISCOVERY_PASSWORD=${NACOS_PASSWORD} + - SPRING_CLOUD_NACOS_CONFIG_USERNAME=${NACOS_USERNAME} + - SPRING_CLOUD_NACOS_CONFIG_PASSWORD=${NACOS_PASSWORD} - SENTINEL_HOST=sentinel - SENTINEL_PORT=8080 + - REDIS_PASSWORD=${REDIS_PASSWORD} + - REDIS_SENTINEL_NODES=redis-sentinel:26379 + - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} + - JWT_SECRET=${USER_JWT_SECRET} + - USER_JWT_SECRET=${USER_JWT_SECRET} + - GATEWAY_IGNORE_URLS=/**/auth/login,/**/auth/register,/**/auth/captcha,/**/auth/refresh,/auth/username/**,/content/video/public/**,/content/category/tree,/content/category/list,/content/tag/list,/content/tag/page,/content/danmaku/all/**,/content/danmaku/time,/content/stat/**,/user/profile/public/**,/user/profile/search,/user/follow/following/**,/user/follow/followers/**,/comment/reply/public/**,/comment/public/**,/recommend/home,/recommend/hot/**,/recommend/similar/**,/recommend/tag/**,/admin/auth/captcha,/admin/auth/login + - JWT_EXPIRATION=${JWT_EXPIRATION} - JAVA_OPTS=-Xms128m -Xmx128m -XX:MetaspaceSize=64m -XX:MaxMetaspaceSize=128m # 添加IP环境变量，用于服务注册 - GATEWAY_IP=${HOST_IP:-127.0.0.1} depends_on: - - nacos + nacos: + condition: service_healthy deploy: resources: limits: @@ -137,19 +295,36 @@ services: container_name: video-auth restart: always environment: - - SPRING_PROFILES_ACTIVE=prod + - SPRING_PROFILES_ACTIVE=ha - NACOS_HOST=nacos - NACOS_PORT=8848 + - NACOS_USERNAME=${NACOS_USERNAME} + - NACOS_PASSWORD=${NACOS_PASSWORD} + - SPRING_CLOUD_NACOS_DISCOVERY_USERNAME=${NACOS_USERNAME} + - SPRING_CLOUD_NACOS_DISCOVERY_PASSWORD=${NACOS_PASSWORD} + - SPRING_CLOUD_NACOS_CONFIG_USERNAME=${NACOS_USERNAME} + - SPRING_CLOUD_NACOS_CONFIG_PASSWORD=${NACOS_PASSWORD} - MYSQL_HOST=mysql - MYSQL_PORT=3306 - - REDIS_HOST=redis - - REDIS_PORT=6379 + - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} + - REDIS_PASSWORD=${REDIS_PASSWORD} + - REDIS_SENTINEL_NODES=redis-sentinel:26379 + - SERVICE_AUTH_TOKEN=${SERVICE_AUTH_TOKEN} + - JWT_SECRET=${USER_JWT_SECRET} + - USER_JWT_SECRET=${USER_JWT_SECRET} + - JWT_EXPIRATION=${JWT_EXPIRATION} + - SENTINEL_DASHBOARD_HOST=sentinel + - SENTINEL_DASHBOARD_PORT=8080 + - SPRING_CLOUD_SENTINEL_TRANSPORT_PORT=8720 # 添加IP环境变量，用于服务注册 - AUTH_IP=${HOST_IP:-127.0.0.1} depends_on: - - nacos - - mysql - - redis + nacos: + condition: service_healthy + mysql: + condition: service_healthy + redis: + condition: service_healthy networks: - video-network @@ -160,19 +335,41 @@ services: container_name: video-user restart: always environment: - - SPRING_PROFILES_ACTIVE=prod + - SPRING_PROFILES_ACTIVE=ha - NACOS_HOST=nacos - NACOS_PORT=8848 + - NACOS_USERNAME=${NACOS_USERNAME} + - NACOS_PASSWORD=${NACOS_PASSWORD} + - SPRING_CLOUD_NACOS_DISCOVERY_USERNAME=${NACOS_USERNAME} + - SPRING_CLOUD_NACOS_DISCOVERY_PASSWORD=${NACOS_PASSWORD} + - SPRING_CLOUD_NACOS_CONFIG_USERNAME=${NACOS_USERNAME} + - SPRING_CLOUD_NACOS_CONFIG_PASSWORD=${NACOS_PASSWORD} - MYSQL_HOST=mysql - MYSQL_PORT=3306 - - REDIS_HOST=redis - - REDIS_PORT=6379 + - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} + - REDIS_PASSWORD=${REDIS_PASSWORD} + - REDIS_SENTINEL_NODES=redis-sentinel:26379 + - MINIO_HOST=minio + - MINIO_PORT=9000 + - MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY} + - MINIO_SECRET_KEY=${MINIO_SECRET_KEY} + - MINIO_BUCKET=${MINIO_BUCKET} + - SERVICE_AUTH_TOKEN=${SERVICE_AUTH_TOKEN} + - JWT_SECRET=${USER_JWT_SECRET} + - USER_JWT_SECRET=${USER_JWT_SECRET} + - JWT_EXPIRATION=${JWT_EXPIRATION} + - SENTINEL_DASHBOARD_HOST=sentinel + - SENTINEL_DASHBOARD_PORT=8080 + - SPRING_CLOUD_SENTINEL_TRANSPORT_PORT=8721 # 添加IP环境变量，用于服务注册 - USER_IP=${HOST_IP:-127.0.0.1} depends_on: - - nacos - - mysql - - redis + nacos: + condition: service_healthy + mysql: + condition: service_healthy + redis: + condition: service_healthy networks: - video-network @@ -183,28 +380,47 @@ services: container_name: video-content restart: always environment: - - SPRING_PROFILES_ACTIVE=prod + - SPRING_PROFILES_ACTIVE=ha - NACOS_HOST=nacos - NACOS_PORT=8848 + - NACOS_USERNAME=${NACOS_USERNAME} + - NACOS_PASSWORD=${NACOS_PASSWORD} + - SPRING_CLOUD_NACOS_DISCOVERY_USERNAME=${NACOS_USERNAME} + - SPRING_CLOUD_NACOS_DISCOVERY_PASSWORD=${NACOS_PASSWORD} + - SPRING_CLOUD_NACOS_CONFIG_USERNAME=${NACOS_USERNAME} + - SPRING_CLOUD_NACOS_CONFIG_PASSWORD=${NACOS_PASSWORD} - MYSQL_HOST=mysql - MYSQL_PORT=3306 - - REDIS_HOST=redis - - REDIS_PORT=6379 + - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} + - REDIS_PASSWORD=${REDIS_PASSWORD} + - REDIS_SENTINEL_NODES=redis-sentinel:26379 - ROCKETMQ_HOST=rmqnamesrv - ROCKETMQ_PORT=9876 - MINIO_HOST=minio - MINIO_PORT=9000 - - MINIO_ACCESS_KEY=minioadmin - - MINIO_SECRET_KEY=minioadmin - - MINIO_BUCKET=video-platform + - MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY} + - MINIO_SECRET_KEY=${MINIO_SECRET_KEY} + - MINIO_BUCKET=${MINIO_BUCKET} + - SERVICE_AUTH_TOKEN=${SERVICE_AUTH_TOKEN} + - JWT_SECRET=${USER_JWT_SECRET} + - USER_JWT_SECRET=${USER_JWT_SECRET} + - JWT_EXPIRATION=${JWT_EXPIRATION} + - SENTINEL_DASHBOARD_HOST=sentinel + - SENTINEL_DASHBOARD_PORT=8080 + - SPRING_CLOUD_SENTINEL_TRANSPORT_PORT=8722 # 添加IP环境变量，用于服务注册 - CONTENT_IP=${HOST_IP:-127.0.0.1} depends_on: - - nacos - - mysql - - redis - - rmqnamesrv - - minio + nacos: + condition: service_healthy + mysql: + condition: service_healthy + redis: + condition: service_healthy + rmqnamesrv: + condition: service_healthy + minio: + condition: service_healthy networks: - video-network @@ -215,19 +431,38 @@ services: container_name: video-admin restart: always environment: - - SPRING_PROFILES_ACTIVE=prod + - SPRING_PROFILES_ACTIVE=ha - NACOS_HOST=nacos - NACOS_PORT=8848 + - NACOS_USERNAME=${NACOS_USERNAME} + - NACOS_PASSWORD=${NACOS_PASSWORD} + - SPRING_CLOUD_NACOS_DISCOVERY_USERNAME=${NACOS_USERNAME} + - SPRING_CLOUD_NACOS_DISCOVERY_PASSWORD=${NACOS_PASSWORD} + - SPRING_CLOUD_NACOS_CONFIG_USERNAME=${NACOS_USERNAME} + - SPRING_CLOUD_NACOS_CONFIG_PASSWORD=${NACOS_PASSWORD} - MYSQL_HOST=mysql - MYSQL_PORT=3306 - - REDIS_HOST=redis - - REDIS_PORT=6379 + - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} + - REDIS_PASSWORD=${REDIS_PASSWORD} + - REDIS_SENTINEL_NODES=redis-sentinel:26379 + - SERVICE_AUTH_TOKEN=${SERVICE_AUTH_TOKEN} + - JWT_SECRET=${ADMIN_JWT_SECRET} + - ADMIN_JWT_SECRET=${ADMIN_JWT_SECRET} + - ADMIN_INIT_PASSWORD=${ADMIN_INIT_PASSWORD} + - APP_INIT_ADMIN_PASSWORD=${ADMIN_INIT_PASSWORD} + - JWT_EXPIRATION=${JWT_EXPIRATION} + - SENTINEL_DASHBOARD_HOST=sentinel + - SENTINEL_DASHBOARD_PORT=8080 + - SPRING_CLOUD_SENTINEL_TRANSPORT_PORT=8725 # 添加IP环境变量，用于服务注册 - ADMIN_IP=${HOST_IP:-127.0.0.1} depends_on: - - nacos - - mysql - - redis + nacos: + condition: service_healthy + mysql: + condition: service_healthy + redis: + condition: service_healthy networks: - video-network @@ -238,19 +473,36 @@ services: container_name: video-comment restart: always environment: - - SPRING_PROFILES_ACTIVE=prod + - SPRING_PROFILES_ACTIVE=ha - NACOS_HOST=nacos - NACOS_PORT=8848 + - NACOS_USERNAME=${NACOS_USERNAME} + - NACOS_PASSWORD=${NACOS_PASSWORD} + - SPRING_CLOUD_NACOS_DISCOVERY_USERNAME=${NACOS_USERNAME} + - SPRING_CLOUD_NACOS_DISCOVERY_PASSWORD=${NACOS_PASSWORD} + - SPRING_CLOUD_NACOS_CONFIG_USERNAME=${NACOS_USERNAME} + - SPRING_CLOUD_NACOS_CONFIG_PASSWORD=${NACOS_PASSWORD} - MYSQL_HOST=mysql - MYSQL_PORT=3306 - - REDIS_HOST=redis - - REDIS_PORT=6379 + - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} + - REDIS_PASSWORD=${REDIS_PASSWORD} + - REDIS_SENTINEL_NODES=redis-sentinel:26379 + - SERVICE_AUTH_TOKEN=${SERVICE_AUTH_TOKEN} + - JWT_SECRET=${USER_JWT_SECRET} + - USER_JWT_SECRET=${USER_JWT_SECRET} + - JWT_EXPIRATION=${JWT_EXPIRATION} + - SENTINEL_DASHBOARD_HOST=sentinel + - SENTINEL_DASHBOARD_PORT=8080 + - SPRING_CLOUD_SENTINEL_TRANSPORT_PORT=8723 # 添加IP环境变量，用于服务注册 - COMMENT_IP=${HOST_IP:-127.0.0.1} depends_on: - - nacos - - mysql - - redis + nacos: + condition: service_healthy + mysql: + condition: service_healthy + redis: + condition: service_healthy networks: - video-network @@ -261,19 +513,35 @@ services: container_name: video-recommend restart: always environment: - - SPRING_PROFILES_ACTIVE=prod + - SPRING_PROFILES_ACTIVE=ha - NACOS_HOST=nacos - NACOS_PORT=8848 + - NACOS_USERNAME=${NACOS_USERNAME} + - NACOS_PASSWORD=${NACOS_PASSWORD} + - SPRING_CLOUD_NACOS_DISCOVERY_USERNAME=${NACOS_USERNAME} + - SPRING_CLOUD_NACOS_DISCOVERY_PASSWORD=${NACOS_PASSWORD} + - SPRING_CLOUD_NACOS_CONFIG_USERNAME=${NACOS_USERNAME} + - SPRING_CLOUD_NACOS_CONFIG_PASSWORD=${NACOS_PASSWORD} - MYSQL_HOST=mysql - MYSQL_PORT=3306 - - REDIS_HOST=redis - - REDIS_PORT=6379 + - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} + - REDIS_PASSWORD=${REDIS_PASSWORD} + - REDIS_SENTINEL_NODES=redis-sentinel:26379 + - JWT_SECRET=${USER_JWT_SECRET} + - USER_JWT_SECRET=${USER_JWT_SECRET} + - JWT_EXPIRATION=${JWT_EXPIRATION} + - SENTINEL_DASHBOARD_HOST=sentinel + - SENTINEL_DASHBOARD_PORT=8080 + - SPRING_CLOUD_SENTINEL_TRANSPORT_PORT=8724 # 添加IP环境变量，用于服务注册 - RECOMMEND_IP=${HOST_IP:-127.0.0.1} depends_on: - - nacos - - mysql - - redis + nacos: + condition: service_healthy + mysql: + condition: service_healthy + redis: + condition: service_healthy networks: - video-network @@ -284,4 +552,9 @@ networks: volumes: mysql_data: redis_data: - minio_data: \ No newline at end of file + redis_sentinel_data: + minio_data: + rmqbroker_logs: + rmqbroker_store: + prometheus_data: + grafana_data: diff --git a/backend/monitoring/README.md b/backend/monitoring/README.md new file mode 100644 index 0000000..42a6f0f --- /dev/null +++ b/backend/monitoring/README.md @@ -0,0 +1,40 @@ +# Monitoring + +最小可用监控栈包含： + +- `Prometheus` +- `Grafana` +- `mysqld-exporter` +- `redis-exporter` +- `node-exporter` +- Spring Boot Actuator `prometheus` 指标端点 + +目录说明： + +- `prometheus/prometheus.local.yml` + 单机 Docker Compose 联调抓取配置 +- `prometheus/prometheus.ha.yml` + 三节点 HA 环境建议的集中抓取配置（默认放在 Node-01） +- `grafana/provisioning` + Grafana 自动配置数据源与仪表盘目录 +- `grafana/dashboards/ha-ovp-overview.json` + 平台总览面板 + +单机访问： + +- Prometheus: `http://localhost:9090` +- Grafana: `http://localhost:3000` + +默认 Grafana 账号密码由 Compose 中环境变量控制： + +- 用户名：`admin` +- 密码：`admin123` + +当前总览面板已包含： + +- 各服务健康状态表 +- 总 QPS / P95 响应时间 +- 各服务请求速率 / 5xx 错误率 +- JVM 堆内存与 GC 概览 +- MySQL / Redis exporter 状态与关键指标 +- Node Exporter 主机 CPU / 内存占位图 diff --git a/backend/monitoring/grafana/dashboards/ha-ovp-overview.json b/backend/monitoring/grafana/dashboards/ha-ovp-overview.json new file mode 100644 index 0000000..a5ab8c0 --- /dev/null +++ b/backend/monitoring/grafana/dashboards/ha-ovp-overview.json @@ -0,0 +1,887 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "text": "DOWN" + }, + "1": { + "color": "green", + "text": "UP" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "editorMode": "code", + "expr": "up{job!=\"prometheus\"}", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Service Health Status", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "mode": "columns" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true + }, + "indexByName": { + "Value": 2, + "application": 0, + "instance": 1, + "job": 3, + "node": 4 + }, + "renameByName": { + "Value": "status" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "targets": [ + { + "editorMode": "code", + "expr": "sum(up{job!=\"prometheus\"})", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Healthy Targets", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "background" + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(http_server_requests_seconds_count{application!=\"\"}[1m]))", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Total QPS", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "background" + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "targets": [ + { + "editorMode": "code", + "expr": "avg(system_cpu_usage{application!=\"\"})", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Average CPU Usage", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "background" + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 0 + }, + "id": 5, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "targets": [ + { + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(http_server_requests_seconds_bucket{application!=\"\"}[5m])) by (le)) * 1000", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "P95 Response Time", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 5 + }, + "id": 6, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "sum by (application) (rate(http_server_requests_seconds_count{application!=\"\"}[1m]))", + "legendFormat": "{{application}}", + "range": true, + "refId": "A" + } + ], + "title": "Request Rate By Service", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 5 + }, + "id": 7, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (application, le) (rate(http_server_requests_seconds_bucket{application!=\"\"}[5m]))) * 1000", + "legendFormat": "{{application}}", + "range": true, + "refId": "A" + } + ], + "title": "P95 Latency By Service", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 8, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "sum by (application) (rate(http_server_requests_seconds_count{application!=\"\",status=~\"5..\"}[5m]))", + "legendFormat": "{{application}}", + "range": true, + "refId": "A" + } + ], + "title": "5xx Error Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 14 + }, + "id": 9, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "sum by (application) (jvm_memory_used_bytes{area=\"heap\",application!=\"\"})", + "legendFormat": "{{application}}", + "range": true, + "refId": "A" + } + ], + "title": "JVM Heap Memory Used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 14 + }, + "id": 10, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "sum by (application) (rate(jvm_gc_pause_seconds_sum{application!=\"\"}[5m]))", + "legendFormat": "{{application}} GC pause/s", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum by (application) (rate(jvm_gc_pause_seconds_count{application!=\"\"}[5m]))", + "legendFormat": "{{application}} GC count/s", + "range": true, + "refId": "B" + } + ], + "title": "JVM GC Overview", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "text": "DOWN" + }, + "1": { + "color": "green", + "text": "UP" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 19 + }, + "id": 11, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "targets": [ + { + "editorMode": "code", + "expr": "max(mysql_up)", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "MySQL Exporter Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 19 + }, + "id": 12, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "targets": [ + { + "editorMode": "code", + "expr": "max(mysql_global_status_threads_connected)", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "MySQL Threads Connected", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "text": "DOWN" + }, + "1": { + "color": "green", + "text": "UP" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 19 + }, + "id": 13, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "targets": [ + { + "editorMode": "code", + "expr": "max(redis_up)", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Redis Exporter Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 19 + }, + "id": 14, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "targets": [ + { + "editorMode": "code", + "expr": "max(redis_memory_used_bytes)", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Redis Memory Used", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 15, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "avg by (instance) (rate(node_cpu_seconds_total{mode!=\"idle\"}[5m]))", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Node CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 16, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Node Memory Usage", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 39, + "style": "dark", + "tags": [ + "ha-ovp", + "spring-boot", + "monitoring" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "HA-OVP Overview", + "uid": "ha-ovp-overview", + "version": 2, + "weekStart": "" +} diff --git a/backend/monitoring/grafana/provisioning/dashboards/dashboard.yml b/backend/monitoring/grafana/provisioning/dashboards/dashboard.yml new file mode 100644 index 0000000..75dcc3d --- /dev/null +++ b/backend/monitoring/grafana/provisioning/dashboards/dashboard.yml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: + - name: HA-OVP + orgId: 1 + folder: HA-OVP + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards diff --git a/backend/monitoring/grafana/provisioning/datasources/datasource.yml b/backend/monitoring/grafana/provisioning/datasources/datasource.yml new file mode 100644 index 0000000..00f9915 --- /dev/null +++ b/backend/monitoring/grafana/provisioning/datasources/datasource.yml @@ -0,0 +1,10 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false diff --git a/backend/monitoring/prometheus/prometheus.ha.yml b/backend/monitoring/prometheus/prometheus.ha.yml new file mode 100644 index 0000000..391c791 --- /dev/null +++ b/backend/monitoring/prometheus/prometheus.ha.yml @@ -0,0 +1,207 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: + - prometheus:9090 + + - job_name: node-01-gateway + metrics_path: /actuator/prometheus + static_configs: + - targets: + - 192.168.1.101:9000 + labels: + application: video-gateway + node: node-01 + + - job_name: node-01-auth + metrics_path: /actuator/prometheus + static_configs: + - targets: + - 192.168.1.101:9001 + labels: + application: video-auth + node: node-01 + + - job_name: node-01-user + metrics_path: /actuator/prometheus + static_configs: + - targets: + - 192.168.1.101:9002 + labels: + application: video-user + node: node-01 + + - job_name: node-01-content + metrics_path: /actuator/prometheus + static_configs: + - targets: + - 192.168.1.101:9003 + labels: + application: video-content + node: node-01 + + - job_name: node-01-comment + metrics_path: /actuator/prometheus + static_configs: + - targets: + - 192.168.1.101:9004 + labels: + application: video-comment + node: node-01 + + - job_name: node-01-recommend + metrics_path: /actuator/prometheus + static_configs: + - targets: + - 192.168.1.101:9005 + labels: + application: video-recommend + node: node-01 + + - job_name: node-01-admin + metrics_path: /actuator/prometheus + static_configs: + - targets: + - 192.168.1.101:9006 + labels: + application: video-admin + node: node-01 + + - job_name: node-02-gateway + metrics_path: /actuator/prometheus + static_configs: + - targets: + - 192.168.1.102:9000 + labels: + application: video-gateway + node: node-02 + + - job_name: node-02-auth + metrics_path: /actuator/prometheus + static_configs: + - targets: + - 192.168.1.102:9001 + labels: + application: video-auth + node: node-02 + + - job_name: node-02-user + metrics_path: /actuator/prometheus + static_configs: + - targets: + - 192.168.1.102:9002 + labels: + application: video-user + node: node-02 + + - job_name: node-02-content + metrics_path: /actuator/prometheus + static_configs: + - targets: + - 192.168.1.102:9003 + labels: + application: video-content + node: node-02 + + - job_name: node-02-comment + metrics_path: /actuator/prometheus + static_configs: + - targets: + - 192.168.1.102:9004 + labels: + application: video-comment + node: node-02 + + - job_name: node-02-recommend + metrics_path: /actuator/prometheus + static_configs: + - targets: + - 192.168.1.102:9005 + labels: + application: video-recommend + node: node-02 + + - job_name: node-02-admin + metrics_path: /actuator/prometheus + static_configs: + - targets: + - 192.168.1.102:9006 + labels: + application: video-admin + node: node-02 + + - job_name: node-01-mysql-exporter + static_configs: + - targets: + - 192.168.1.101:9104 + labels: + application: mysql + node: node-01 + + - job_name: node-02-mysql-exporter + static_configs: + - targets: + - 192.168.1.102:9104 + labels: + application: mysql + node: node-02 + + - job_name: node-03-mysql-exporter + static_configs: + - targets: + - 192.168.1.103:9104 + labels: + application: mysql + node: node-03 + + - job_name: node-01-redis-exporter + static_configs: + - targets: + - 192.168.1.101:9121 + labels: + application: redis + node: node-01 + + - job_name: node-02-redis-exporter + static_configs: + - targets: + - 192.168.1.102:9121 + labels: + application: redis + node: node-02 + + - job_name: node-03-redis-exporter + static_configs: + - targets: + - 192.168.1.103:9121 + labels: + application: redis + node: node-03 + + - job_name: node-01-node-exporter + static_configs: + - targets: + - 192.168.1.101:9100 + labels: + application: node + node: node-01 + + - job_name: node-02-node-exporter + static_configs: + - targets: + - 192.168.1.102:9100 + labels: + application: node + node: node-02 + + - job_name: node-03-node-exporter + static_configs: + - targets: + - 192.168.1.103:9100 + labels: + application: node + node: node-03 diff --git a/backend/monitoring/prometheus/prometheus.local.yml b/backend/monitoring/prometheus/prometheus.local.yml new file mode 100644 index 0000000..34a1a72 --- /dev/null +++ b/backend/monitoring/prometheus/prometheus.local.yml @@ -0,0 +1,86 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: + - prometheus:9090 + + - job_name: video-gateway + metrics_path: /actuator/prometheus + static_configs: + - targets: + - video-gateway:9000 + labels: + application: video-gateway + + - job_name: video-auth + metrics_path: /actuator/prometheus + static_configs: + - targets: + - video-auth:9001 + labels: + application: video-auth + + - job_name: video-user + metrics_path: /actuator/prometheus + static_configs: + - targets: + - video-user:9002 + labels: + application: video-user + + - job_name: video-content + metrics_path: /actuator/prometheus + static_configs: + - targets: + - video-content:9003 + labels: + application: video-content + + - job_name: video-comment + metrics_path: /actuator/prometheus + static_configs: + - targets: + - video-comment:9004 + labels: + application: video-comment + + - job_name: video-recommend + metrics_path: /actuator/prometheus + static_configs: + - targets: + - video-recommend:9005 + labels: + application: video-recommend + + - job_name: video-admin + metrics_path: /actuator/prometheus + static_configs: + - targets: + - video-admin:9006 + labels: + application: video-admin + + - job_name: mysql-exporter + static_configs: + - targets: + - mysqld-exporter:9104 + labels: + application: mysql + + - job_name: redis-exporter + static_configs: + - targets: + - redis-exporter:9121 + labels: + application: redis + + - job_name: node-exporter + static_configs: + - targets: + - node-exporter:9100 + labels: + application: node diff --git a/backend/pom.xml b/backend/pom.xml index 59719cc..d076b3c 100644 --- a/backend/pom.xml +++ b/backend/pom.xml @@ -92,6 +92,7 @@ 3.25.3 test + @@ -128,4 +129,4 @@ - \ No newline at end of file + diff --git a/backend/video-admin/pom.xml b/backend/video-admin/pom.xml index 17d9a11..a4ae1c9 100644 --- a/backend/video-admin/pom.xml +++ b/backend/video-admin/pom.xml @@ -49,6 +49,11 @@ spring-boot-starter-actuator + + io.micrometer + micrometer-registry-prometheus + + org.springframework.boot @@ -149,4 +154,4 @@ - \ No newline at end of file + diff --git a/backend/video-auth/pom.xml b/backend/video-auth/pom.xml index 5d0b0db..001b87a 100644 --- a/backend/video-auth/pom.xml +++ b/backend/video-auth/pom.xml @@ -56,6 +56,11 @@ spring-boot-starter-actuator + + io.micrometer + micrometer-registry-prometheus + + org.springframework.boot @@ -137,4 +142,4 @@ - \ No newline at end of file + diff --git a/backend/video-comment/pom.xml b/backend/video-comment/pom.xml index 25ad03e..3f43175 100644 --- a/backend/video-comment/pom.xml +++ b/backend/video-comment/pom.xml @@ -49,6 +49,11 @@ spring-boot-starter-actuator + + io.micrometer + micrometer-registry-prometheus + + org.springframework.boot @@ -124,4 +129,4 @@ - \ No newline at end of file + diff --git a/backend/video-content/pom.xml b/backend/video-content/pom.xml index 29d2388..b47539e 100644 --- a/backend/video-content/pom.xml +++ b/backend/video-content/pom.xml @@ -49,6 +49,11 @@ spring-boot-starter-actuator + + io.micrometer + micrometer-registry-prometheus + + org.springframework.boot @@ -138,4 +143,4 @@ - \ No newline at end of file + diff --git a/backend/video-gateway/pom.xml b/backend/video-gateway/pom.xml index d5e4816..48df2e2 100644 --- a/backend/video-gateway/pom.xml +++ b/backend/video-gateway/pom.xml @@ -54,6 +54,16 @@ spring-boot-starter-actuator + + io.micrometer + micrometer-registry-prometheus + + + + org.springframework.boot + spring-boot-starter-data-redis-reactive + + org.projectlombok @@ -104,4 +114,4 @@ - \ No newline at end of file + diff --git a/backend/video-gateway/src/main/java/com/nuaa/video/gateway/filter/AuthGlobalFilter.java b/backend/video-gateway/src/main/java/com/nuaa/video/gateway/filter/AuthGlobalFilter.java index c954396..9833fb0 100644 --- a/backend/video-gateway/src/main/java/com/nuaa/video/gateway/filter/AuthGlobalFilter.java +++ b/backend/video-gateway/src/main/java/com/nuaa/video/gateway/filter/AuthGlobalFilter.java @@ -12,6 +12,7 @@ import org.springframework.cloud.gateway.filter.GatewayFilterChain; import org.springframework.cloud.gateway.filter.GlobalFilter; import org.springframework.core.Ordered; import org.springframework.core.io.buffer.DataBuffer; +import org.springframework.data.redis.core.ReactiveStringRedisTemplate; import org.springframework.http.HttpStatus; import org.springframework.http.server.reactive.ServerHttpRequest; import org.springframework.http.server.reactive.ServerHttpResponse; @@ -35,17 +36,20 @@ import java.util.Map; @RequiredArgsConstructor public class AuthGlobalFilter implements GlobalFilter, Ordered { - @Value("${jwt.secret:video-platform-secret-key-2024-nuaa}") + private static final String USER_ACCESS_TOKEN_PREFIX = "user:access-token:"; + + @Value("${jwt.secret}") private String secret; @Value("${jwt.token-header:Authorization}") private String tokenHeader; - @Value("${gateway.ignore.urls:/**/auth/login,/**/auth/register,/**/auth/captcha,/content/video/public/**,/content/category/tree,/content/category/list,/comment/reply/public/**,/comment/public/**,/admin/auth/captcha,/admin/auth/login}") + @Value("${gateway.ignore.urls:/**/auth/login,/**/auth/register,/**/auth/captcha,/**/auth/refresh,/auth/username/**,/content/video/public/**,/content/category/tree,/content/category/list,/content/tag/list,/content/tag/page,/content/danmaku/all/**,/content/danmaku/time,/content/stat/**,/user/profile/public/**,/user/profile/search,/user/follow/following/**,/user/follow/followers/**,/comment/reply/public/**,/comment/public/**,/recommend/home,/recommend/hot/**,/recommend/similar/**,/recommend/tag/**,/admin/auth/captcha,/admin/auth/login}") private List ignoreUrls; private final AntPathMatcher pathMatcher = new AntPathMatcher(); private final ObjectMapper objectMapper = new ObjectMapper(); + private final ReactiveStringRedisTemplate redisTemplate; @Override public Mono filter(ServerWebExchange exchange, GatewayFilterChain chain) { @@ -63,15 +67,29 @@ public class AuthGlobalFilter implements GlobalFilter, Ordered { return unauthorized(exchange, "缺少token"); } + if (path.startsWith("/admin/")) { + return chain.filter(exchange); + } + try { // 验证token Claims claims = parseToken(token); + return redisTemplate.hasKey(USER_ACCESS_TOKEN_PREFIX + token) + .flatMap(exists -> { + if (!Boolean.TRUE.equals(exists)) { + return unauthorized(exchange, "Token已失效"); + } - // 将用户信息传递到下游服务 - ServerHttpRequest.Builder mutate = request.mutate(); - mutate.header("userId", claims.getSubject()); - mutate.header("username", claims.get("username", String.class)); - return chain.filter(exchange.mutate().request(mutate.build()).build()); + // 将用户信息传递到下游服务 + ServerHttpRequest.Builder mutate = request.mutate(); + mutate.header("userId", claims.getSubject()); + mutate.header("username", claims.get("username", String.class)); + return chain.filter(exchange.mutate().request(mutate.build()).build()); + }) + .onErrorResume(e -> { + log.error("Redis校验token失败", e); + return unauthorized(exchange, "认证服务暂不可用"); + }); } catch (Exception e) { log.error("Token验证失败", e); return unauthorized(exchange, "Token验证失败"); @@ -141,4 +159,4 @@ public class AuthGlobalFilter implements GlobalFilter, Ordered { return response.setComplete(); } } -} \ No newline at end of file +} diff --git a/backend/video-gateway/src/main/resources/application-prod.yml b/backend/video-gateway/src/main/resources/application-prod.yml index c44083a..c96ab60 100644 --- a/backend/video-gateway/src/main/resources/application-prod.yml +++ b/backend/video-gateway/src/main/resources/application-prod.yml @@ -4,6 +4,12 @@ server: spring: application: name: video-gateway + data: + redis: + host: ${REDIS_HOST:redis} + port: ${REDIS_PORT:6379} + password: ${REDIS_PASSWORD} + database: 0 cloud: nacos: discovery: @@ -106,4 +112,8 @@ spring: args: retries: 3 statuses: BAD_GATEWAY,SERVICE_UNAVAILABLE - methods: GET,POST \ No newline at end of file + methods: GET,POST + +jwt: + secret: ${USER_JWT_SECRET} + token-header: Authorization diff --git a/backend/video-gateway/src/main/resources/application.yml b/backend/video-gateway/src/main/resources/application.yml index 96dd6cb..c58e833 100644 --- a/backend/video-gateway/src/main/resources/application.yml +++ b/backend/video-gateway/src/main/resources/application.yml @@ -9,6 +9,7 @@ spring: config: import: - optional:nacos:common.yaml?refreshEnabled=true + - optional:nacos:common-redis-${spring.profiles.active}.yaml?refreshEnabled=true - optional:nacos:${spring.application.name}.yaml?refreshEnabled=true cloud: nacos: diff --git a/backend/video-recommend/pom.xml b/backend/video-recommend/pom.xml index 365b81e..0160219 100644 --- a/backend/video-recommend/pom.xml +++ b/backend/video-recommend/pom.xml @@ -49,6 +49,11 @@ spring-boot-starter-actuator + + io.micrometer + micrometer-registry-prometheus + + org.springframework.boot @@ -131,4 +136,4 @@ - \ No newline at end of file + diff --git a/backend/video-user/pom.xml b/backend/video-user/pom.xml index 7e6ffcf..8271bcc 100644 --- a/backend/video-user/pom.xml +++ b/backend/video-user/pom.xml @@ -49,6 +49,11 @@ spring-boot-starter-actuator + + io.micrometer + micrometer-registry-prometheus + + org.springframework.boot @@ -143,4 +148,4 @@ - \ No newline at end of file + diff --git a/docs/DEBIAN12-DEPLOYMENT-GUIDE.md b/docs/DEBIAN12-DEPLOYMENT-GUIDE.md index c970b01..b2f967b 100644 --- a/docs/DEBIAN12-DEPLOYMENT-GUIDE.md +++ b/docs/DEBIAN12-DEPLOYMENT-GUIDE.md @@ -102,6 +102,11 @@ ssh node-03 "hostname" ## 二、部署步骤 +说明： +- 多虚拟机 HA 部署以 `docker-compose-node-01.yml`、`docker-compose-node-02.yml`、`docker-compose-node-03.yml` 为准。 +- 微服务端口以各模块的 `application.yml` 和 Nacos 中的 `video-*.yaml` 为准。 +- 根目录的 `backend/docker-compose.yml` 仅用于单机本地联调，允许在基础设施编排方式上与 HA 版不同，但服务端口和数据库编号应保持一致。 + ### 2.1 克隆项目 建议提前配置好 GitHub 账号和 SSH 密钥 @@ -121,7 +126,7 @@ cd HA-OVP # 构建所有模块 mvn clean install -Dmaven.test.skip=true -# 构建 Sentinel 镜像 (三个节点都需要执行) +# 构建 Sentinel 镜像（Node-01 执行一次即可） cd sentinel-dashboard docker build -t sentinel-dashboard:1.8.6 . cd .. diff --git a/docs/HA-DEPLOYMENT-GUIDE.md b/docs/HA-DEPLOYMENT-GUIDE.md deleted file mode 100644 index 57459bd..0000000 --- a/docs/HA-DEPLOYMENT-GUIDE.md +++ /dev/null @@ -1,339 +0,0 @@ -# 高可用集群部署手册 - -## 一、环境准备 - -### 1.1 虚拟机配置要求 - -| 节点 | IP地址 | CPU | 内存 | 磁盘 | 操作系统 | -|------|--------|-----|------|------|----------| -| Node-01 | 192.168.1.101 | 4核 | 8GB | 100GB | Ubuntu 22.04 / CentOS 8 | -| Node-02 | 192.168.1.102 | 4核 | 8GB | 100GB | Ubuntu 22.04 / CentOS 8 | -| Node-03 | 192.168.1.103 | 4核 | 8GB | 100GB | Ubuntu 22.04 / CentOS 8 | - -### 1.2 软件依赖 - -每台虚拟机需要安装以下软件： - -```bash -# Ubuntu -apt-get update -apt-get install -y docker.io docker-compose-plugin openjdk-17-jdk maven git curl wget - -# CentOS -yum install -y docker docker-compose-plugin java-17-openjdk-devel maven git curl wget - -# 启动Docker -systemctl start docker -systemctl enable docker - -# 添加当前用户到docker组 -usermod -aG docker $USER -``` - -### 1.3 网络配置 - -确保三台虚拟机之间网络互通： - -```bash -# 在每台机器上测试 -ping 192.168.1.101 -ping 192.168.1.102 -ping 192.168.1.103 - -# 开放防火墙端口 -# Ubuntu -ufw allow 3306,6379,8848,9000,9001,9876,10911,8880,9000:9006/tcp - -# CentOS -firewall-cmd --permanent --add-port={3306,6379,8848,9000,9001,9876,10911,8880,9000-9006}/tcp -firewall-cmd --reload -``` - ---- - -## 二、部署步骤 - -### 2.1 克隆项目到所有节点 - -```bash -# 在所有节点执行 -git clone <项目地址> /opt/ha-ovp -cd /opt/ha-ovp -``` - -### 2.2 构建项目 - -```bash -# 在任意节点构建（只需执行一次） -cd /opt/ha-ovp - -# 构建common模块 -cd video-common -mvn clean install -DskipTests -cd .. - -# 构建所有服务 -mvn clean package -DskipTests - -# 构建Sentinel镜像 -cd sentinel-dashboard -docker build -t sentinel-dashboard:1.8.6 . -cd .. - -# 将构建产物分发到其他节点 -scp -r target/ video-*/target/ root@192.168.1.102:/opt/ha-ovp/ -scp -r target/ video-*/target/ root@192.168.1.103:/opt/ha-ovp/ -``` - -### 2.3 初始化各节点 - -```bash -# Node-01 (192.168.1.101) -chmod +x deploy-ha.sh -./deploy-ha.sh 1 init - -# Node-02 (192.168.1.102) -chmod +x deploy-ha.sh -./deploy-ha.sh 2 init - -# Node-03 (192.168.1.103) -chmod +x deploy-ha.sh -./deploy-ha.sh 3 init -``` - -### 2.4 按顺序启动服务 - -#### 步骤1: 启动基础设施 (所有节点) - -```bash -# Node-01 先启动 -./deploy-ha.sh 1 infra - -# 等待Node-01完全启动后 (约2分钟) -# Node-02 启动 -./deploy-ha.sh 2 infra - -# Node-03 启动 -./deploy-ha.sh 3 infra -``` - -#### 步骤2: 配置MySQL主从复制 - -```bash -# Node-01 查看主节点状态 -docker exec mysql mysql -uroot -proot123456 -e "SHOW MASTER STATUS\G" - -# 记录 File 和 Position 值 - -# Node-02 配置从节点 -docker exec mysql mysql -uroot -proot123456 <; -START SLAVE; -SHOW SLAVE STATUS\G -EOF - -# Node-03 配置从节点 (同上) -``` - -#### 步骤3: 配置Redis主从复制 - -```bash -# Node-02 和 Node-03 执行 -docker exec redis redis-cli -a redis_HA_Cluster_2024 REPLICAOF 192.168.1.101 6379 - -# 验证 -docker exec redis redis-cli -a redis_HA_Cluster_2024 INFO replication -``` - -#### 步骤4: 配置Nacos集群 - -访问 http://192.168.1.101:8848/nacos (账号: nacos/nacos) - -创建命名空间 `public`，并添加以下配置文件： - -| Data ID | Group | 说明 | -|---------|-------|------| -| video-gateway.yaml | DEFAULT_GROUP | 网关配置 | -| video-auth.yaml | DEFAULT_GROUP | 认证服务配置 | -| video-user.yaml | DEFAULT_GROUP | 用户服务配置 | -| video-content.yaml | DEFAULT_GROUP | 内容服务配置 | -| video-comment.yaml | DEFAULT_GROUP | 评论服务配置 | -| video-recommend.yaml | DEFAULT_GROUP | 推荐服务配置 | -| video-admin.yaml | DEFAULT_GROUP | 管理后台配置 | - -#### 步骤5: 初始化MinIO - -```bash -# 任意节点执行 -docker run --rm --network ha-network minio/mc < status - -# 检查Nacos服务注册 -curl http://192.168.1.101:8848/nacos/v1/ns/instance/list?serviceName=video-gateway - -# 检查Redis主从 -docker exec redis redis-cli -a redis_HA_Cluster_2024 INFO replication - -# 检查MySQL主从 -docker exec mysql mysql -uroot -proot123456 -e "SHOW SLAVE STATUS\G" -``` - -### 3.2 访问服务 - -| 服务 | 地址 | -|------|------| -| Nacos控制台 | http://192.168.1.101:8848/nacos | -| Sentinel控制台 | http://192.168.1.101:8880 | -| MinIO控制台 | http://192.168.1.101:9101 | -| RocketMQ控制台 | http://192.168.1.102:8080 (需额外部署) | -| API网关 | http://192.168.1.100 (通过Nginx) | - -### 3.3 高可用测试 - -#### 测试服务故障转移 - -```bash -# 停止Node-01上的网关服务 -docker stop video-gateway - -# 访问API验证（应该自动切换到Node-02） -curl http://192.168.1.100/auth/captcha - -# 恢复服务 -docker start video-gateway -``` - -#### 测试Redis主从切换 - -```bash -# 停止Node-01的Redis -docker stop redis - -# Sentinel会自动将从节点提升为主节点 -# 查看Sentinel日志 -docker logs redis-sentinel - -# 恢复Redis -docker start redis -``` - -#### 测试MySQL主从切换 - -```bash -# 停止主节点MySQL -docker stop mysql - -# 查看从节点状态 -docker exec mysql mysql -uroot -proot123456 -e "SHOW SLAVE STATUS\G" -``` - ---- - -## 四、常见问题 - -### Q1: Nacos集群无法启动 - -检查各节点时间是否同步： - -```bash -# 同步时间 -apt-get install -y ntpdate -ntpdate pool.ntp.org -``` - -### Q2: 服务注册不到Nacos - -检查网络连通性： - -```bash -# 测试Nacos连接 -curl http://192.168.1.101:8848/nacos/v1/console/health/liveness -``` - -### Q3: Redis连接失败 - -检查Redis密码配置是否一致： - -```bash -# 测试连接 -docker exec redis redis-cli -a redis_HA_Cluster_2024 ping -``` - -### Q4: MySQL主从同步失败 - -```bash -# 重置从节点 -docker exec mysql mysql -uroot -proot123456 <