diff --git a/backend/config/.env.node-01 b/backend/config/.env.node-01 index 8e59790..d73dade 100644 --- a/backend/config/.env.node-01 +++ b/backend/config/.env.node-01 @@ -39,8 +39,9 @@ SENTINEL_HOST=sentinel SENTINEL_PORT=8080 # MinIO配置 -MINIO_HOST=minio -MINIO_PORT=9000 +# 方案A:所有上传统一写入 Node-01,本节点既是写入点也是读取点 +MINIO_HOST=192.168.1.101 +MINIO_PORT=9100 MINIO_EXTERNAL_PORT=9100 MINIO_ACCESS_KEY=replace-with-minio-access-key MINIO_SECRET_KEY=replace-with-minio-secret-key diff --git a/backend/config/.env.node-02 b/backend/config/.env.node-02 index 09573f2..d2a9499 100644 --- a/backend/config/.env.node-02 +++ b/backend/config/.env.node-02 @@ -39,8 +39,9 @@ SENTINEL_HOST=sentinel SENTINEL_PORT=8080 # MinIO配置 -MINIO_HOST=minio -MINIO_PORT=9000 +# 方案A:所有上传统一写入 Node-01,Node-02 仅作为同步副本 +MINIO_HOST=192.168.1.101 +MINIO_PORT=9100 MINIO_EXTERNAL_PORT=9100 MINIO_ACCESS_KEY=replace-with-minio-access-key MINIO_SECRET_KEY=replace-with-minio-secret-key diff --git a/backend/docker-compose-node-01.yml b/backend/docker-compose-node-01.yml index 3784c1e..3652100 100644 --- a/backend/docker-compose-node-01.yml +++ b/backend/docker-compose-node-01.yml @@ -425,8 +425,8 @@ services: - REDIS_SENTINEL_NODES=${REDIS_SENTINEL_NODES:-192.168.1.101:26379,192.168.1.102:26379,192.168.1.103:26379} - ROCKETMQ_HOST=rmqnamesrv - ROCKETMQ_PORT=9876 - - MINIO_HOST=minio - - MINIO_PORT=9000 + - MINIO_HOST=192.168.1.101 + - MINIO_PORT=9100 - MINIO_ACCESS_KEY=minioadmin - MINIO_SECRET_KEY=minioadmin123 - MINIO_BUCKET=video-platform diff --git a/backend/docker-compose-node-02.yml b/backend/docker-compose-node-02.yml index 4d3bf6f..c2d775e 100644 --- a/backend/docker-compose-node-02.yml +++ b/backend/docker-compose-node-02.yml @@ -331,6 +331,8 @@ services: - MYSQL_PASSWORD=${MYSQL_ROOT_PASSWORD:-root123456} - REDIS_PASSWORD=${REDIS_PASSWORD:-redis_HA_Cluster_2024} - REDIS_SENTINEL_NODES=${REDIS_SENTINEL_NODES:-192.168.1.101:26379,192.168.1.102:26379,192.168.1.103:26379} + - MINIO_HOST=192.168.1.101 + - MINIO_PORT=9100 - MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY} - MINIO_SECRET_KEY=${MINIO_SECRET_KEY} - MINIO_BUCKET=${MINIO_BUCKET} diff --git a/backend/import_category_videos.ps1 b/backend/import_category_videos.ps1 new file mode 100644 index 0000000..07ff6bf --- /dev/null +++ b/backend/import_category_videos.ps1 @@ -0,0 +1,763 @@ +param( + [string]$VideoRoot = 'D:\视频库', + [string]$BaseUrl = 'http://192.168.1.101:9000', + [string]$OutputPath = '', + [string]$AdminUsername = 'admin', + [string]$AdminPassword = 'admin123', + [string]$DefaultUserPassword = 'Pass@123456', + [string]$RedisHost = '192.168.1.101', + [int]$RedisPort = 6379, + [string]$RedisPassword = '', + [switch]$ReuseExistingUsers +) + +$ErrorActionPreference = 'Stop' + +if ([string]::IsNullOrWhiteSpace($OutputPath)) { + $OutputPath = Join-Path $PSScriptRoot 'VIDEO_IMPORT_RESULTS.json' +} + +function New-OrderedMap { + return [ordered]@{} +} + +function ConvertTo-PlainObject { + param([object]$Value) + + if ($null -eq $Value) { + return $null + } + + if ($Value -is [System.Collections.IDictionary]) { + $result = [ordered]@{} + foreach ($key in $Value.Keys) { + $result[$key] = ConvertTo-PlainObject -Value $Value[$key] + } + return $result + } + + if ($Value -is [System.Collections.IEnumerable] -and -not ($Value -is [string])) { + $items = @() + foreach ($item in $Value) { + $items += ,(ConvertTo-PlainObject -Value $item) + } + return $items + } + + $type = $Value.GetType() + if ($type.FullName -like 'System.Management.Automation.PSCustomObject' -or + ($type.Namespace -and $type.Namespace -notlike 'System*')) { + $result = [ordered]@{} + foreach ($prop in $Value.PSObject.Properties) { + $result[$prop.Name] = ConvertTo-PlainObject -Value $prop.Value + } + return $result + } + + return $Value +} + +function Save-Report { + param([hashtable]$Report) + + $plain = ConvertTo-PlainObject -Value $Report + $plain | ConvertTo-Json -Depth 20 | Set-Content -Path $OutputPath -Encoding utf8BOM +} + +function Invoke-JsonApi { + param( + [Parameter(Mandatory = $true)][string]$Method, + [Parameter(Mandatory = $true)][string]$Url, + [hashtable]$Headers = @{}, + $Body = $null, + [int]$TimeoutSec = 180 + ) + + $params = @{ + Method = $Method + Uri = $Url + Headers = $Headers + UseBasicParsing = $true + SkipHttpErrorCheck = $true + TimeoutSec = $TimeoutSec + } + + if ($null -ne $Body) { + $params.ContentType = 'application/json' + if ($Body -is [string]) { + $params.Body = $Body + } else { + $params.Body = ($Body | ConvertTo-Json -Depth 20) + } + } + + try { + $response = Invoke-WebRequest @params + $json = $null + if ($response.Content) { + try { + $json = $response.Content | ConvertFrom-Json -Depth 50 + } catch { + $json = $null + } + } + + return [pscustomobject]@{ + HttpStatus = [int]$response.StatusCode + Json = $json + Raw = $response.Content + } + } catch { + return [pscustomobject]@{ + HttpStatus = -1 + Json = $null + Raw = $_.Exception.Message + } + } +} + +function Invoke-UploadApi { + param( + [Parameter(Mandatory = $true)][string]$Url, + [Parameter(Mandatory = $true)][string]$FilePath, + [hashtable]$Headers = @{}, + [int]$TimeoutSec = 7200 + ) + + $handler = $null + $httpClient = $null + $fileStream = $null + try { + $handler = New-Object System.Net.Http.HttpClientHandler + $httpClient = New-Object System.Net.Http.HttpClient($handler) + $httpClient.Timeout = [TimeSpan]::FromSeconds($TimeoutSec) + + foreach ($headerKey in $Headers.Keys) { + [void]$httpClient.DefaultRequestHeaders.TryAddWithoutValidation($headerKey, [string]$Headers[$headerKey]) + } + + $multipart = New-Object System.Net.Http.MultipartFormDataContent + $fileStream = [System.IO.File]::OpenRead($FilePath) + $fileContent = New-Object System.Net.Http.StreamContent($fileStream) + + $mimeType = switch ([System.IO.Path]::GetExtension($FilePath).ToLowerInvariant()) { + '.mp4' { 'video/mp4' } + '.mov' { 'video/quicktime' } + '.mkv' { 'video/x-matroska' } + '.avi' { 'video/x-msvideo' } + '.webm' { 'video/webm' } + '.m4v' { 'video/x-m4v' } + '.flv' { 'video/x-flv' } + default { 'video/mp4' } + } + $fileContent.Headers.ContentType = [System.Net.Http.Headers.MediaTypeHeaderValue]::Parse($mimeType) + $multipart.Add($fileContent, 'file', [System.IO.Path]::GetFileName($FilePath)) + + $response = $httpClient.PostAsync($Url, $multipart).GetAwaiter().GetResult() + $raw = $response.Content.ReadAsStringAsync().GetAwaiter().GetResult() + $json = $null + if ($raw) { + try { + $json = $raw | ConvertFrom-Json -Depth 50 + } catch { + $json = $null + } + } + + return [pscustomobject]@{ + HttpStatus = [int]$response.StatusCode + Json = $json + Raw = $raw + } + } catch { + return [pscustomobject]@{ + HttpStatus = -1 + Json = $null + Raw = $_.Exception.Message + } + } finally { + if ($fileStream) { $fileStream.Dispose() } + if ($httpClient) { $httpClient.Dispose() } + if ($handler) { $handler.Dispose() } + } +} + +function Read-DotEnv { + param([string]$Path) + + $map = @{} + if (-not (Test-Path -LiteralPath $Path)) { + return $map + } + + foreach ($line in Get-Content -LiteralPath $Path) { + if ([string]::IsNullOrWhiteSpace($line) -or $line.TrimStart().StartsWith('#')) { + continue + } + $parts = $line -split '=', 2 + if ($parts.Count -eq 2) { + $map[$parts[0].Trim()] = $parts[1].Trim() + } + } + return $map +} + +function Read-RedisLine { + param([System.IO.Stream]$Stream) + + $bytes = New-Object System.Collections.Generic.List[byte] + while ($true) { + $b = $Stream.ReadByte() + if ($b -lt 0) { + throw 'Redis 连接已关闭' + } + if ($b -eq 13) { + $next = $Stream.ReadByte() + if ($next -ne 10) { + throw 'Redis 响应格式错误' + } + break + } + $bytes.Add([byte]$b) + } + return [System.Text.Encoding]::UTF8.GetString($bytes.ToArray()) +} + +function Send-RedisCommand { + param( + [System.IO.Stream]$Stream, + [string[]]$RedisArgs + ) + + $builder = New-Object System.Text.StringBuilder + [void]$builder.Append("*$($RedisArgs.Count)`r`n") + foreach ($arg in $RedisArgs) { + $bytes = [System.Text.Encoding]::UTF8.GetBytes($arg) + [void]$builder.Append("$" + $bytes.Length + "`r`n") + [void]$builder.Append($arg + "`r`n") + } + + $payload = [System.Text.Encoding]::UTF8.GetBytes($builder.ToString()) + $Stream.Write($payload, 0, $payload.Length) + $Stream.Flush() + + $prefix = [char]$Stream.ReadByte() + switch ($prefix) { + '+' { return Read-RedisLine -Stream $Stream } + '-' { throw "Redis 错误: $(Read-RedisLine -Stream $Stream)" } + ':' { return [int64](Read-RedisLine -Stream $Stream) } + '$' { + $length = [int](Read-RedisLine -Stream $Stream) + if ($length -lt 0) { + return $null + } + $buffer = New-Object byte[] $length + $offset = 0 + while ($offset -lt $length) { + $read = $Stream.Read($buffer, $offset, $length - $offset) + if ($read -le 0) { + throw 'Redis bulk string 读取失败' + } + $offset += $read + } + $null = Read-RedisLine -Stream $Stream + return [System.Text.Encoding]::UTF8.GetString($buffer) + } + '*' { + $count = [int](Read-RedisLine -Stream $Stream) + $items = @() + for ($i = 0; $i -lt $count; $i++) { + $nestedPrefix = [char]$Stream.ReadByte() + if ($nestedPrefix -ne '$') { + throw "暂不支持的 Redis 数组子类型: $nestedPrefix" + } + $len = [int](Read-RedisLine -Stream $Stream) + if ($len -lt 0) { + $items += $null + continue + } + $buf = New-Object byte[] $len + $offset = 0 + while ($offset -lt $len) { + $read = $Stream.Read($buf, $offset, $len - $offset) + if ($read -le 0) { + throw 'Redis 数组元素读取失败' + } + $offset += $read + } + $null = Read-RedisLine -Stream $Stream + $items += [System.Text.Encoding]::UTF8.GetString($buf) + } + return $items + } + default { throw "未知 Redis 响应前缀: $prefix" } + } +} + +function Get-AvailableRedisPasswords { + $candidates = New-Object System.Collections.Generic.List[string] + foreach ($candidate in @( + $RedisPassword, + 'redis_HA_Cluster_2024', + 'replace-with-strong-redis-password', + 'redis_EsJ8FX' + )) { + if (-not [string]::IsNullOrWhiteSpace($candidate) -and -not $candidates.Contains($candidate)) { + $candidates.Add($candidate) + } + } + return $candidates +} + +function Get-RedisValue { + param( + [Parameter(Mandatory = $true)][string]$Key, + [int]$RedisDbIndex = 0 + ) + + $passwords = Get-AvailableRedisPasswords + $lastError = $null + + foreach ($passwordCandidate in $passwords) { + $tcp = $null + try { + $tcp = [System.Net.Sockets.TcpClient]::new() + $tcp.ReceiveTimeout = 5000 + $tcp.SendTimeout = 5000 + $tcp.Connect($RedisHost, $RedisPort) + $stream = $tcp.GetStream() + + [void](Send-RedisCommand -Stream $stream -RedisArgs @('AUTH', $passwordCandidate)) + if ($RedisDbIndex -gt 0) { + [void](Send-RedisCommand -Stream $stream -RedisArgs @('SELECT', $RedisDbIndex.ToString())) + } + $value = Send-RedisCommand -Stream $stream -RedisArgs @('GET', $Key) + $tcp.Close() + return $value + } catch { + $lastError = $_.Exception.Message + if ($tcp) { + $tcp.Close() + } + } + } + + throw "读取 Redis 键失败: $Key;host=${RedisHost}:$RedisPort;lastError=$lastError" +} + +function Get-CaptchaCode { + param( + [Parameter(Mandatory = $true)][string]$CaptchaUrl, + [Parameter(Mandatory = $true)][string]$RedisKeyPrefix, + [int]$RedisDb = 0 + ) + + $captchaResp = Invoke-JsonApi -Method GET -Url $CaptchaUrl + if ($captchaResp.HttpStatus -ne 200 -or -not $captchaResp.Json -or -not $captchaResp.Json.data) { + throw "获取验证码失败: $($captchaResp.Raw)" + } + + $captchaKey = [string]$captchaResp.Json.data.captchaKey + $code = Get-RedisValue -Key "$RedisKeyPrefix$captchaKey" -RedisDbIndex $RedisDb + + return [pscustomobject]@{ + CaptchaKey = $captchaKey + CaptchaCode = ([string]$code).Trim('"').Trim() + } +} + +function Ensure-ApiSuccess { + param( + [Parameter(Mandatory = $true)]$Response, + [Parameter(Mandatory = $true)][string]$Action + ) + + if ($Response.HttpStatus -ne 200) { + throw "$Action 失败,HTTP=$($Response.HttpStatus),响应=$($Response.Raw)" + } + + if (-not $Response.Json) { + throw "$Action 失败,响应不是 JSON: $($Response.Raw)" + } + + if ($Response.Json.code -ne 200) { + throw "$Action 失败,API code=$($Response.Json.code),message=$($Response.Json.message)" + } +} + +function New-RandomPhone { + $digits = -join ((0..9) | Get-Random -Count 9) + return "13$digits" +} + +function New-Username { + param( + [string]$CategoryName, + [int]$Index + ) + + $stamp = (Get-Date).ToString('MMddHHmm') + return ("u{0}{1:D2}" -f $stamp, $Index) +} + +function Normalize-VideoTitle { + param([string]$FileNameWithoutExtension) + + $title = $FileNameWithoutExtension.Trim() + if ($title.Length -lt 5) { + $title = ($title + ' 视频素材').Substring(0, [Math]::Min(($title + ' 视频素材').Length, 100)) + } + if ($title.Length -gt 100) { + $title = $title.Substring(0, 100) + } + return $title +} + +function Get-UserSession { + param( + [string]$Username, + [string]$Password + ) + + $captcha = Get-CaptchaCode -CaptchaUrl "$BaseUrl/auth/captcha" -RedisKeyPrefix 'captcha:' -RedisDb 0 + $loginResp = Invoke-JsonApi -Method POST -Url "$BaseUrl/auth/login" -Body @{ + username = $Username + password = $Password + captcha = $captcha.CaptchaCode + captchaKey = $captcha.CaptchaKey + } + Ensure-ApiSuccess -Response $loginResp -Action "用户登录($Username)" + + $accessToken = [string]$loginResp.Json.data.accessToken + $headers = @{ Authorization = "Bearer $accessToken" } + $currentResp = Invoke-JsonApi -Method GET -Url "$BaseUrl/auth/current" -Headers $headers + Ensure-ApiSuccess -Response $currentResp -Action "读取当前用户($Username)" + + return [pscustomobject]@{ + Username = $Username + Password = $Password + UserId = [long]$currentResp.Json.data.id + Headers = $headers + AccessToken = $accessToken + } +} + +function Get-ReusedUsersByCategory { + param([string]$ReportPath) + + if (-not (Test-Path -LiteralPath $ReportPath)) { + throw "复用用户失败,报告文件不存在: $ReportPath" + } + + $json = Get-Content -LiteralPath $ReportPath -Raw | ConvertFrom-Json -Depth 50 + $map = @{} + foreach ($entry in @($json.users)) { + $categoryName = [string]$entry.categoryName + if (-not $map.ContainsKey($categoryName)) { + $map[$categoryName] = New-Object System.Collections.Generic.List[object] + } + $session = Get-UserSession -Username ([string]$entry.username) -Password ([string]$entry.password) + $session | Add-Member -NotePropertyName Nickname -NotePropertyValue ([string]$entry.nickname) -Force + $session | Add-Member -NotePropertyName CategoryName -NotePropertyValue $categoryName -Force + $map[$categoryName].Add($session) + } + return $map +} + +function Register-ImportUser { + param( + [string]$CategoryName, + [int]$UserIndex, + [string]$Nickname + ) + + $username = New-Username -CategoryName $CategoryName -Index $UserIndex + $phone = New-RandomPhone + $email = "$username@example.com" + $captcha = Get-CaptchaCode -CaptchaUrl "$BaseUrl/auth/captcha" -RedisKeyPrefix 'captcha:' -RedisDb 0 + + $registerResp = Invoke-JsonApi -Method POST -Url "$BaseUrl/auth/register" -Body @{ + username = $username + password = $DefaultUserPassword + confirmPassword = $DefaultUserPassword + email = $email + phone = $phone + captcha = $captcha.CaptchaCode + captchaKey = $captcha.CaptchaKey + } + Ensure-ApiSuccess -Response $registerResp -Action "注册用户($username)" + + $session = Get-UserSession -Username $username -Password $DefaultUserPassword + + $profileResp = Invoke-JsonApi -Method PUT -Url "$BaseUrl/user/profile" -Headers $session.Headers -Body @{ + nickname = $Nickname + signature = "批量导入账号 - $CategoryName" + region = '南京' + gender = 0 + } + Ensure-ApiSuccess -Response $profileResp -Action "更新资料($username)" + + $session | Add-Member -NotePropertyName Nickname -NotePropertyValue $Nickname + $session | Add-Member -NotePropertyName CategoryName -NotePropertyValue $CategoryName + return $session +} + +function Get-AdminSession { + $captcha = Get-CaptchaCode -CaptchaUrl "$BaseUrl/admin/auth/captcha" -RedisKeyPrefix 'admin:captcha:' -RedisDb 5 + $loginResp = Invoke-JsonApi -Method POST -Url "$BaseUrl/admin/auth/login" -Body @{ + username = $AdminUsername + password = $AdminPassword + captcha = $captcha.CaptchaCode + captchaKey = $captcha.CaptchaKey + } + Ensure-ApiSuccess -Response $loginResp -Action '管理员登录' + + $token = [string]$loginResp.Json.data.token + $headers = @{ Authorization = "Bearer $token" } + $infoResp = Invoke-JsonApi -Method GET -Url "$BaseUrl/admin/auth/info" -Headers $headers + Ensure-ApiSuccess -Response $infoResp -Action '获取管理员信息' + + return [pscustomobject]@{ + Headers = $headers + AdminId = [long]$infoResp.Json.data.id + Username = $AdminUsername + } +} + +function Get-CategoryMap { + $resp = Invoke-JsonApi -Method GET -Url "$BaseUrl/content/category/list" + Ensure-ApiSuccess -Response $resp -Action '读取分类列表' + + $map = @{} + foreach ($item in @($resp.Json.data)) { + $map[[string]$item.name] = $item + } + return $map +} + +function Get-VideoFilesForFolder { + param([string]$FolderPath) + + $extensions = @('.mp4', '.mov', '.mkv', '.avi', '.flv', '.webm', '.m4v') + return @(Get-ChildItem -LiteralPath $FolderPath -File | + Where-Object { $extensions -contains $_.Extension.ToLowerInvariant() } | + Sort-Object Name) +} + +$report = [ordered]@{ + generatedAt = (Get-Date).ToString('s') + baseUrl = $BaseUrl + videoRoot = $VideoRoot + outputPath = $OutputPath + userPassword = $DefaultUserPassword + summary = [ordered]@{ + categoryCount = 0 + expectedUsers = 0 + createdUsers = 0 + expectedVideos = 0 + uploadedVideos = 0 + reviewedVideos = 0 + failedVideos = 0 + } + categories = @() + users = @() + errors = @() +} + +if (-not (Test-Path -LiteralPath $VideoRoot)) { + throw "视频根目录不存在: $VideoRoot" +} + +$categoryFolders = @(Get-ChildItem -LiteralPath $VideoRoot -Directory | Sort-Object Name) +if ($categoryFolders.Count -ne 5) { + throw "期望在 $VideoRoot 下找到 5 个分类目录,实际找到 $($categoryFolders.Count) 个" +} + +$categoryMap = Get-CategoryMap +$adminSession = Get-AdminSession +$reusedUsersByCategory = $null +if ($ReuseExistingUsers) { + $reusedUsersByCategory = Get-ReusedUsersByCategory -ReportPath $OutputPath + $report.users = @() + foreach ($categoryUsers in $reusedUsersByCategory.Values) { + foreach ($user in $categoryUsers) { + $report.users += [ordered]@{ + categoryName = $user.CategoryName + username = $user.Username + nickname = $user.Nickname + password = $user.Password + userId = $user.UserId + assignedVideoCount = $null + } + } + } + $report.summary.createdUsers = $report.users.Count +} + +Save-Report -Report $report + +$report.summary.categoryCount = $categoryFolders.Count +$report.summary.expectedUsers = $categoryFolders.Count * 2 +$report.summary.expectedVideos = $categoryFolders.Count * 5 +$report.summary.uploadedVideos = 0 +$report.summary.reviewedVideos = 0 +$report.summary.failedVideos = 0 +$report.categories = @() +$report.errors = @() + +$globalUserIndex = 1 + +foreach ($folder in $categoryFolders) { + $categoryName = $folder.Name + if (-not $categoryMap.ContainsKey($categoryName)) { + throw "平台中未找到分类: $categoryName" + } + + $videoFiles = Get-VideoFilesForFolder -FolderPath $folder.FullName + if ($videoFiles.Count -ne 5) { + throw "分类 [$categoryName] 期望有 5 个视频,实际找到 $($videoFiles.Count) 个" + } + + $platformCategory = $categoryMap[$categoryName] + $categoryReport = [ordered]@{ + categoryName = $categoryName + categoryId = [long]$platformCategory.id + folderPath = $folder.FullName + userAssignments = @() + } + + $report.categories += $categoryReport + Save-Report -Report $report + + $assignmentSpecs = @( + @{ UserLabel = 'A'; VideoIndexes = @(0, 1); VideoCount = 2 }, + @{ UserLabel = 'B'; VideoIndexes = @(2, 3, 4); VideoCount = 3 } + ) + + $reusedCategoryUsers = $null + if ($ReuseExistingUsers) { + if (-not $reusedUsersByCategory.ContainsKey($categoryName) -or $reusedUsersByCategory[$categoryName].Count -ne 2) { + throw "分类 [$categoryName] 没有可复用的 2 个用户" + } + $reusedCategoryUsers = $reusedUsersByCategory[$categoryName].ToArray() + } + + $assignmentIndex = 0 + foreach ($assignmentSpec in $assignmentSpecs) { + if ($ReuseExistingUsers) { + $userSession = $reusedCategoryUsers[$assignmentIndex] + $nickname = $userSession.Nickname + } else { + $nickname = "$categoryName-导入用户$($assignmentSpec.UserLabel)" + $userSession = Register-ImportUser -CategoryName $categoryName -UserIndex $globalUserIndex -Nickname $nickname + $globalUserIndex++ + + $report.summary.createdUsers++ + $report.users += [ordered]@{ + categoryName = $categoryName + username = $userSession.Username + nickname = $nickname + password = $userSession.Password + userId = $userSession.UserId + assignedVideoCount = $assignmentSpec.VideoCount + } + } + + $userAssignmentReport = [ordered]@{ + userId = $userSession.UserId + username = $userSession.Username + nickname = $nickname + assignedVideoCount = $assignmentSpec.VideoCount + videos = @() + } + + foreach ($videoIndex in $assignmentSpec.VideoIndexes) { + $file = $videoFiles[$videoIndex] + $videoReport = [ordered]@{ + sourceFile = $file.FullName + fileName = $file.Name + ownerUserId = $userSession.UserId + ownerUsername = $userSession.Username + status = 'processing' + upload = $null + create = $null + review = $null + publicCheck = $null + error = $null + } + + try { + $uploadResp = Invoke-UploadApi -Url "$BaseUrl/content/video/upload" -FilePath $file.FullName -Headers $userSession.Headers + Ensure-ApiSuccess -Response $uploadResp -Action "上传视频($($file.Name))" + + $videoTitle = Normalize-VideoTitle -FileNameWithoutExtension $file.BaseName + $createResp = Invoke-JsonApi -Method POST -Url "$BaseUrl/content/video" -Headers $userSession.Headers -Body @{ + title = $videoTitle + description = "批量导入视频;分类:$categoryName;源文件:$($file.Name)" + categoryId = [long]$platformCategory.id + tagIds = @() + coverUrl = $uploadResp.Json.data.coverUrl + videoUrl = $uploadResp.Json.data.videoUrl + duration = $uploadResp.Json.data.duration + width = $uploadResp.Json.data.width + height = $uploadResp.Json.data.height + format = $uploadResp.Json.data.format + size = $uploadResp.Json.data.size + } + Ensure-ApiSuccess -Response $createResp -Action "创建视频($($file.Name))" + + $videoId = [long]$createResp.Json.data + $reviewResp = Invoke-JsonApi -Method POST -Url "$BaseUrl/admin/platform/content/video/review/$videoId?status=1&reason=bulk-import" -Headers $adminSession.Headers + Ensure-ApiSuccess -Response $reviewResp -Action "审核视频($videoId)" + + $publicCheckResp = Invoke-JsonApi -Method GET -Url "$BaseUrl/content/video/public/$videoId" + Ensure-ApiSuccess -Response $publicCheckResp -Action "校验公开视频($videoId)" + + $videoReport.status = 'completed' + $videoReport.upload = [ordered]@{ + videoUrl = $uploadResp.Json.data.videoUrl + coverUrl = $uploadResp.Json.data.coverUrl + duration = $uploadResp.Json.data.duration + width = $uploadResp.Json.data.width + height = $uploadResp.Json.data.height + format = $uploadResp.Json.data.format + size = $uploadResp.Json.data.size + } + $videoReport.create = [ordered]@{ + videoId = $videoId + title = $videoTitle + categoryId = [long]$platformCategory.id + } + $videoReport.review = [ordered]@{ + status = 1 + reviewerAdminId = $adminSession.AdminId + } + $videoReport.publicCheck = [ordered]@{ + videoId = $videoId + apiCode = $publicCheckResp.Json.code + } + + $report.summary.uploadedVideos++ + $report.summary.reviewedVideos++ + } catch { + $videoReport.status = 'failed' + $videoReport.error = $_.Exception.Message + $report.summary.failedVideos++ + $report.errors += "[$categoryName][$($userSession.Username)][$($file.Name)] $($_.Exception.Message)" + } + + $userAssignmentReport.videos += $videoReport + Save-Report -Report $report + } + + $categoryReport.userAssignments += $userAssignmentReport + $assignmentIndex++ + Save-Report -Report $report + } + Save-Report -Report $report +} + +$report.generatedAt = (Get-Date).ToString('s') +Save-Report -Report $report + +$report | ConvertTo-Json -Depth 20 diff --git a/backend/scripts/minio-sync-node01-to-node02.sh b/backend/scripts/minio-sync-node01-to-node02.sh new file mode 100644 index 0000000..4cad09b --- /dev/null +++ b/backend/scripts/minio-sync-node01-to-node02.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -euo pipefail + +SRC_ALIAS="${SRC_ALIAS:-node01}" +DST_ALIAS="${DST_ALIAS:-node02}" +SRC_ENDPOINT="${SRC_ENDPOINT:-http://192.168.1.101:9100}" +DST_ENDPOINT="${DST_ENDPOINT:-http://192.168.1.102:9100}" +MINIO_USER="${MINIO_USER:-minioadmin}" +MINIO_PASSWORD="${MINIO_PASSWORD:-minioadmin123}" +BUCKET="${BUCKET:-video-platform}" +MC_BIN="${MC_BIN:-/tmp/mc}" + +if [ ! -x "$MC_BIN" ]; then + curl -L -o "$MC_BIN" https://dl.min.io/client/mc/release/linux-amd64/mc + chmod +x "$MC_BIN" +fi + +"$MC_BIN" alias set "$SRC_ALIAS" "$SRC_ENDPOINT" "$MINIO_USER" "$MINIO_PASSWORD" >/dev/null +"$MC_BIN" alias set "$DST_ALIAS" "$DST_ENDPOINT" "$MINIO_USER" "$MINIO_PASSWORD" >/dev/null + +"$MC_BIN" mirror --overwrite "$SRC_ALIAS/$BUCKET" "$DST_ALIAS/$BUCKET" diff --git a/backend/sql/init.sql b/backend/sql/init.sql index d151b4b..66650c8 100644 --- a/backend/sql/init.sql +++ b/backend/sql/init.sql @@ -148,7 +148,7 @@ CREATE TABLE IF NOT EXISTS `user_info` ( `following_count` int NOT NULL DEFAULT '0' COMMENT '关注数', `video_count` int NOT NULL DEFAULT '0' COMMENT '视频数', `like_count` int NOT NULL DEFAULT '0' COMMENT '获赞数', - `coin_count` int NOT NULL DEFAULT '0' COMMENT '硬币数', + `coin_count` int NOT NULL DEFAULT '20' COMMENT '硬币数', `favorite_count` int NOT NULL DEFAULT '0' COMMENT '收藏数', `view_count` bigint NOT NULL DEFAULT '0' COMMENT '播放数', `level` tinyint NOT NULL DEFAULT '1' COMMENT '等级', diff --git a/backend/sql/nacos-config-init.sql b/backend/sql/nacos-config-init.sql index 05dd09b..d382ad6 100644 --- a/backend/sql/nacos-config-init.sql +++ b/backend/sql/nacos-config-init.sql @@ -105,6 +105,11 @@ spring: INSERT INTO config_info (data_id, group_id, content, tenant_id, app_name, src_user, encrypted_data_key) VALUES ('video-gateway.yaml', 'DEFAULT_GROUP', '# Gateway 服务配置 spring: + codec: + max-in-memory-size: 10MB + webflux: + multipart: + max-in-memory-size: 10MB cloud: gateway: discovery: @@ -120,10 +125,18 @@ spring: uri: lb://video-user predicates: - Path=/user/** + filters: + - name: RequestSize + args: + maxSize: 10MB - id: video-content uri: lb://video-content predicates: - Path=/content/** + filters: + - name: RequestSize + args: + maxSize: 1024MB - id: video-comment uri: lb://video-comment predicates: @@ -219,6 +232,10 @@ spring: data: redis: database: 1 + servlet: + multipart: + max-file-size: 10MB + max-request-size: 10MB cloud: sentinel: transport: diff --git a/backend/video-gateway/src/main/resources/application.yml b/backend/video-gateway/src/main/resources/application.yml index c58e833..aa20cb6 100644 --- a/backend/video-gateway/src/main/resources/application.yml +++ b/backend/video-gateway/src/main/resources/application.yml @@ -27,6 +27,11 @@ spring: file-extension: yaml username: ${NACOS_USERNAME:nacos} password: ${NACOS_PASSWORD:nacos} + codec: + max-in-memory-size: ${GATEWAY_MAX_IN_MEMORY_SIZE:10MB} + webflux: + multipart: + max-in-memory-size: ${GATEWAY_MULTIPART_MAX_IN_MEMORY_SIZE:10MB} management: endpoints: diff --git a/backend/video-user/src/main/resources/application.yml b/backend/video-user/src/main/resources/application.yml index c822a17..1e6661f 100644 --- a/backend/video-user/src/main/resources/application.yml +++ b/backend/video-user/src/main/resources/application.yml @@ -27,6 +27,14 @@ spring: file-extension: yaml username: ${NACOS_USERNAME:nacos} password: ${NACOS_PASSWORD:nacos} + servlet: + multipart: + max-file-size: ${USER_AVATAR_MAX_FILE_SIZE:10MB} + max-request-size: ${USER_AVATAR_MAX_REQUEST_SIZE:10MB} + +user: + avatar: + max-size-bytes: ${USER_AVATAR_MAX_SIZE_BYTES:5242880} management: endpoints: diff --git a/docs/Debian 12 高可用集群部署手册.md b/docs/Debian 12 高可用集群部署手册.md new file mode 100644 index 0000000..7f3abe9 --- /dev/null +++ b/docs/Debian 12 高可用集群部署手册.md @@ -0,0 +1,1110 @@ +# Debian 12 高可用集群部署手册 + +## 一、环境准备 + +### 1.1 虚拟机配置要求 + +| 节点 | IP地址 | CPU | 内存 | 磁盘 | 操作系统 | +|------|--------|-----|------|------|----------| +| Node-01 | 192.168.1.101 | 4核 | 8GB | 100GB | Debian 12 (Bookworm) | +| Node-02 | 192.168.1.102 | 4核 | 8GB | 100GB | Debian 12 (Bookworm) | +| Node-03 | 192.168.1.103 | 4核 | 8GB | 100GB | Debian 12 (Bookworm) | + +### 1.2 Debian 12 安装建议 + +下载地址: https://www.debian.org/distrib/ + +推荐使用 **Debian 12 Netinst** 最小化安装: +- 选择 "无图形界面" 安装 +- 仅安装 SSH server 和标准系统工具 +- 使用 LVM 分区,便于扩展 + +### 1.3 系统初始化 + +在每台虚拟机上执行以下操作: + +```bash +# 切换到 root 用户 +su - + +# 上传 init-debian12.sh 脚本并执行 +chmod +x init-debian12.sh +./init-debian12.sh + +# 使环境变量生效 +source /etc/profile + +# 验证安装 +java -version +mvn -version +docker --version +docker compose version +``` + +### 1.4 网络配置 + +#### 配置静态 IP + +```bash +# 编辑网络配置 +vim /etc/network/interfaces + +# 添加以下内容 (根据实际网络调整) +#(VMware虚拟机版) +# Node-01 +# The primary network interface +auto ens33 +iface ens33 inet static + address 192.168.1.101 + netmask 255.255.255.0 + gateway 192.168.1.2 + dns-nameservers 8.8.8.8 114.114.114.114 + +# 重启网络 +systemctl restart networking +reboot +``` + +#### 配置主机名 + +```bash +# Node-01 +hostnamectl set-hostname node-01 +echo "192.168.1.101 node-01" >> /etc/hosts +echo "192.168.1.102 node-02" >> /etc/hosts +echo "192.168.1.103 node-03" >> /etc/hosts + +# Node-02 +hostnamectl set-hostname node-02 +# 添加 hosts... + +# Node-03 +hostnamectl set-hostname node-03 +# 添加 hosts... +``` + +#### 配置 SSH 免密登录 + +```bash +# 在 Node-01 生成密钥 +ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa + +# 分发公钥到其他节点 +ssh-copy-id root@192.168.1.102 +ssh-copy-id root@192.168.1.103 + +# 测试连接 +ssh node-02 "hostname" +ssh node-03 "hostname" +``` + +--- + +## 二、部署步骤 + +说明: +- 多虚拟机 HA 部署以 `docker-compose-node-01.yml`、`docker-compose-node-02.yml`、`docker-compose-node-03.yml` 为准。 +- 微服务端口以各模块的 `application.yml` 和 Nacos 中的 `video-*.yaml` 为准。 +- 根目录的 `backend/docker-compose.yml` 仅用于单机本地联调,允许在基础设施编排方式上与 HA 版不同,但服务端口和数据库编号应保持一致。 + +### 2.1 克隆项目 + +建议提前配置好 GitHub 账号和 SSH 密钥 + +```bash +# 在所有节点执行 +cd /opt +git clone <项目地址> HA-OVP +cd HA-OVP +``` + +### 2.2 构建项目 + +```bash +# 在 Node-01 执行 (只需执行一次) + +# 构建所有模块 +mvn clean install -Dmaven.test.skip=true + +# 构建 Sentinel 镜像(Node-01 执行一次即可) +cd sentinel-dashboard +docker build -t sentinel-dashboard:1.8.6 . +cd .. + +# 验证构建产物 +ls -la video-*/target/*.jar +``` + +### 2.3 同步构建产物 + +```bash +# 三个节点都需要安装 rsync +apt install rsync -y + +# 从 Node-01 同步到其他节点 +rsync -avz --progress /opt/HA-OVP/ root@192.168.1.102:/opt/HA-OVP/ +rsync -avz --progress /opt/HA-OVP/ root@192.168.1.103:/opt/HA-OVP/ +``` + +### 2.4 Node-01 部署 (主节点) + +#### 步骤 1: 启动基础设施 + +```bash +cd /opt/HA-OVP/backend + +# 先只启动基础设施 (MySQL, Redis, Nacos 等) +docker compose -f docker-compose-node-01.yml up -d mysql redis redis-sentinel nacos rmqnamesrv minio sentinel + +# 等待 MySQL 完全启动 (约 30-60 秒) +docker compose -f docker-compose-node-01.yml logs -f mysql +# 看到 "ready for connections" 后 Ctrl+C 退出 + +#停止 +docker compose -f docker-compose-node-01.yml down -v +``` + +#### 步骤 2: 验证数据库初始化 + +MySQL 容器启动时会自动执行 `sql/` 目录下的初始化脚本(通过 `docker-entrypoint-initdb.d` 挂载),无需手动初始化。 + +```bash +# 验证数据库已自动创建 +docker exec mysql mysql -uroot -proot123456 -e "SHOW DATABASES;" + +# 应看到以下数据库: +# - video_auth +# - video_user +# - video_content +# - video_comment +# - video_recommend +# - video_admin +# - nacos_config +``` + +#### 步骤 3: 等待 Nacos 启动 + +```bash +# 等待 Nacos 完全启动 +docker compose -f docker-compose-node-01.yml logs -f nacos +# 看到 "Nacos started successfully" 后 Ctrl+C 退出 + +# 创建 Nacos 用户 (重要!微服务需要认证) +# 注意:如果是首次启动,Nacos 默认用户是 nacos/nacos +# 访问控制台验证: http://192.168.1.101:8848/nacos +``` + +#### 步骤 4: 启动微服务 + +```bash +# 启动所有微服务 +docker compose -f docker-compose-node-01.yml up -d video-gateway video-auth video-user video-content video-comment video-recommend video-admin nginx + +# 查看微服务启动日志 +docker compose -f docker-compose-node-01.yml logs -f video-gateway +# 确认日志中没有错误,显示 "Started GatewayApplication" 表示成功 +``` + +#### 步骤 5: 验证 Node-01 服务状态 + +```bash +# 检查所有容器状态 +docker ps + +# 验证 Nacos 服务注册 +curl http://192.168.1.101:8848/nacos/v1/ns/service/list | grep -o '"name":"[^"]*"' + +# 验证微服务健康状态 +curl http://192.168.1.101:9000/actuator/health +curl http://192.168.1.101:9001/actuator/health +``` + +### 2.5 Node-02 部署 (从节点) + +**重要**:确保 Node-01 的所有服务正常运行后再部署 Node-02 + +#### 步骤 1: 启动基础设施 + +```bash +cd /opt/HA-OVP/backend + +# 启动基础设施 +docker compose -f docker-compose-node-02.yml up -d mysql redis redis-sentinel nacos rmqnamesrv rmqbroker minio sentinel + +# 等待 MySQL 和 Nacos 完全启动 +docker compose -f docker-compose-node-02.yml logs -f mysql +# 看到 "ready for connections" 后 Ctrl+C 退出 +``` + +#### 步骤 2: 启动微服务 + +**注意**: +- Node-02 作为 MySQL 从节点,数据库会通过主从复制自动同步,无需手动初始化。 +- 对象存储采用“方案A:Node-01 统一写入,Node-02 同步副本”,因此 `video-user` 与 `video-content` 的 MinIO 上传目标必须统一指向 `192.168.1.101:9100`。 +- Node-02 本地 MinIO 保留部署,但不再作为业务随机写入点。 + +```bash +# 等待 Nacos 启动后,启动微服务 +docker compose -f docker-compose-node-02.yml up -d video-gateway video-auth video-user video-content video-comment video-recommend video-admin + +# 查看日志,确认 Nacos 认证成功 (应显示 username='nacos') +docker compose -f docker-compose-node-02.yml logs -f video-gateway +``` + +### 2.6 Node-03 部署 (备份节点) + +```bash +cd /opt/HA-OVP/backend + +# Node-03 只部署基础设施 +docker compose -f docker-compose-node-03.yml up -d + +# 等待 MySQL 和 Nacos 启动 +docker compose -f docker-compose-node-03.yml logs -f mysql +# 看到 "ready for connections" 后 Ctrl+C 退出 + +``` + +### 2.7 配置 MySQL 主从复制 + +#### 在 Node-01 (主节点) 创建复制用户 + +```bash +docker exec -i mysql mysql -uroot -proot123456 << 'EOF' +CREATE USER IF NOT EXISTS 'repl'@'%' IDENTIFIED WITH mysql_native_password BY 'repl_password_2024'; +GRANT REPLICATION SLAVE ON *.* TO 'repl'@'%'; +FLUSH PRIVILEGES; +SHOW MASTER STATUS; +EOF +``` + +#### 在 Node-02 和 Node-03 (从节点) 配置主从 + +```bash +# 在 Node-02 执行 +docker exec -i mysql mysql -uroot -proot123456 << 'EOF' +CHANGE MASTER TO + MASTER_HOST='192.168.1.101', + MASTER_USER='repl', + MASTER_PASSWORD='repl_password_2024', + MASTER_AUTO_POSITION=1; +START SLAVE; +SHOW SLAVE STATUS\G; +EOF + +# 确认 Slave_IO_Running 和 Slave_SQL_Running 都为 Yes + +# 配置完成后,启用只读模式 (重要!) +docker exec mysql mysql -uroot -proot123456 -e "SET GLOBAL read_only = ON; SET GLOBAL super_read_only = ON;" +``` + +```bash +# 在 Node-03 执行 (同上) +docker exec -i mysql mysql -uroot -proot123456 << 'EOF' +CHANGE MASTER TO + MASTER_HOST='192.168.1.101', + MASTER_USER='repl', + MASTER_PASSWORD='repl_password_2024', + MASTER_AUTO_POSITION=1; +START SLAVE; +SHOW SLAVE STATUS\G; +EOF + +docker exec mysql mysql -uroot -proot123456 -e "SET GLOBAL read_only = ON; SET GLOBAL super_read_only = ON;" +``` + +### 2.8 配置 Redis 主从复制 + +```bash +# 在 Node-02 执行 +docker exec redis redis-cli -a redis_HA_Cluster_2024 REPLICAOF 192.168.1.101 6379 + +# 在 Node-03 执行 +docker exec redis redis-cli -a redis_HA_Cluster_2024 REPLICAOF 192.168.1.101 6379 + +# 验证主从状态 (在 Node-01 执行) +docker exec redis redis-cli -a redis_HA_Cluster_2024 INFO replication | grep -E "role|connected_slaves" +``` + +### 2.9 配置 Nacos + +访问 http://192.168.1.101:8848/nacos (账号: nacos/nacos) + +#### 方式一:SQL 脚本初始化(推荐) + +```bash +# 在 Node-01 执行,初始化所有配置到 Nacos 数据库 +docker exec -i mysql mysql -uroot -proot123456 nacos_config < /opt/HA-OVP/sql/nacos-config-init.sql + +# 验证配置是否导入成功 +docker exec mysql mysql -uroot -proot123456 -e "SELECT data_id FROM nacos_config.config_info;" +``` + +SQL 脚本会自动创建以下配置: + +| Data ID | 说明 | +|---------|------| +| common.yaml | 公共配置(MySQL连接池、MyBatis、Feign等) | +| common-redis-dev.yaml | 开发环境 Redis 单机配置 | +| common-redis-ha.yaml | 生产环境 Redis Sentinel 配置 | +| video-gateway.yaml | Gateway 服务专属配置 | +| video-auth.yaml | Auth 服务专属配置 | +| video-user.yaml | User 服务专属配置 | +| video-content.yaml | Content 服务专属配置 | +| video-comment.yaml | Comment 服务专属配置 | +| video-recommend.yaml | Recommend 服务专属配置 | +| video-admin.yaml | Admin 服务专属配置 | + +#### 方式二:控制台手动创建 + +1. **创建命名空间**: 命名空间 → 新建命名空间 → 命名空间ID: `public`, 命名空间名: `公共命名空间` +2. **添加配置文件**: 配置管理 → 配置列表 → 创建配置 + - Data ID: `common.yaml` + - Group: `DEFAULT_GROUP` + - 配置内容: 参考 `sql/nacos-config-init.sql` 中的内容 +3. **重复以上步骤添加其他配置文件** + +#### 配置架构说明 + +配置采用分层加载模式: + +``` +application.yml (本地,极简) +├── common.yaml (公共配置) +├── common-redis-${profile}.yaml (按环境的Redis配置) +└── ${spring.application.name}.yaml (服务专属配置) +``` + +**环境切换**:通过 `SPRING_PROFILES_ACTIVE` 环境变量 +- `dev`: 加载 `common-redis-dev.yaml`(单机Redis) +- `ha`: 加载 `common-redis-ha.yaml`(Sentinel哨兵) + +**配置热更新**:修改 Nacos 中的配置后,服务会自动感知并更新,无需重启! + +### 2.10 初始化 MinIO + +```bash +# 安装 mc 客户端 +wget https://dl.min.io/client/mc/release/linux-amd64/mc +chmod +x mc +mv mc /usr/local/bin/ + +# 配置别名 (注意端口是 9100) +mc alias set myminio http://192.168.1.101:9100 minioadmin minioadmin123 + +# 创建存储桶 +mc mb myminio/video-platform --ignore-existing + +# 设置公开访问策略 +mc anonymous set download myminio/video-platform + +# 验证 +mc ls myminio +``` + +#### 对象存储高可用方案(方案A) + +当前生产部署不采用 MinIO 真正分布式集群模式,而采用更稳定的“单写入点 + 副本同步”方案: + +- `Node-01 MinIO (192.168.1.101:9100)`:唯一业务写入点,也是前端统一读取入口 +- `Node-02 MinIO (192.168.1.102:9100)`:同步副本,仅用于灾备,不作为业务随机写入点 +- 前端媒体访问统一使用 `http://192.168.1.101:9100/video-platform` +- `video-user`、`video-content` 服务的 MinIO 上传目标统一配置为 `MINIO_HOST=192.168.1.101`、`MINIO_PORT=9100` + +#### MinIO 同步脚本 + +仓库已提供同步脚本: + +```bash +/opt/HA-OVP/backend/scripts/minio-sync-node01-to-node02.sh +``` + +脚本作用: + +- 将 `Node-01` 的 `video-platform` 桶内容镜像同步到 `Node-02` +- 适用于头像、封面、视频文件的对象复制 + +#### 配置定时同步任务 + +在 `Node-01` 上配置 cron,每 5 分钟同步一次: + +```bash +crontab -e + +*/5 * * * * /bin/bash /opt/HA-OVP/backend/scripts/minio-sync-node01-to-node02.sh >> /var/log/minio-sync.log 2>&1 +``` + +检查定时任务: + +```bash +crontab -l +tail -f /var/log/minio-sync.log +``` + +#### 手动执行一次同步 + +首次部署完成后,建议在 `Node-01` 手动执行一次: + +```bash +cd /opt/HA-OVP/backend +bash scripts/minio-sync-node01-to-node02.sh +``` + +#### 说明 + +- 该方案可以避免“上传成功但另一节点读取 404”的问题 +- 该方案属于“对象存储主节点写入 + 副本节点同步”的高可用实现 +- 严格意义上,这不是 MinIO 分布式纠删码集群,而是适合当前资源条件的稳定答辩版方案 + +--- + +## 三、端口分配汇总 + +### 3.1 各节点端口分配 + +| 服务 | Node-01 | Node-02 | Node-03 | 说明 | +|------|---------|---------|---------|------| +| **MySQL** | 3306 | 3306 | 3306 | 数据库主从 | +| **Redis** | 6379 | 6379 | 6379 | 缓存主从 | +| **Redis Sentinel** | 26379 | 26379 | 26379 | 哨兵监控 | +| **Nacos** | 8848 | 8848 | 8848 | 配置中心集群 | +| **RocketMQ NameServer** | 9876 | 9876 | 9876 | NameServer集群 | +| **RocketMQ Broker** | - | 10911 | 10911 | Broker双主 | +| **MinIO API** | 9100 | 9100 | 9000 | Node-01 为统一写入与读取入口,Node-02 为同步副本 | +| **MinIO Console** | 9101 | 9101 | 9001 | 对象存储控制台 | +| **Sentinel Dashboard** | 8880 | 8880 | 8880 | 熔断控制台 | +| **video-gateway** | 9000 | 9000 | - | API网关 | +| **video-auth** | 9001 | 9001 | - | 认证服务 | +| **video-user** | 9002 | 9002 | - | 用户服务 | +| **video-content** | 9003 | 9003 | - | 内容服务 | +| **video-comment** | 9004 | 9004 | - | 评论服务 | +| **video-recommend** | 9005 | 9005 | - | 推荐服务 | +| **video-admin** | 9006 | 9006 | - | 管理后台 | +| **Nginx** | 80 | - | - | 负载均衡 | + +### 3.2 端口说明 + +- **MinIO 端口调整**: Node-01 和 Node-02 的 MinIO 端口改为 9100(API) 和 9101(Console),避免与微服务端口冲突 +- **MinIO 访问策略**: 所有业务上传统一写入 Node-01 MinIO,Node-02 通过同步脚本保持副本一致性 +- **Node-03**: 仅部署基础设施,不部署微服务,作为热备份节点 + +--- + +## 四、验证部署 + +### 4.1 检查服务状态 + +```bash +# 检查 Docker 容器状态 +docker ps + +# 检查 Nacos 服务注册 +curl http://192.168.1.101:8848/nacos/v1/ns/service/list + +# 检查 MySQL 主从 +docker exec mysql mysql -uroot -proot123456 -e "SHOW SLAVE STATUS\G" | grep Running + +# 检查 Redis 主从 +docker exec redis redis-cli -a redis_HA_Cluster_2024 INFO replication | grep role + +# 检查 Sentinel 状态 +docker exec redis-sentinel redis-cli -p 26379 SENTINEL master mymaster +``` + +### 4.2 访问控制台 + +| 服务 | 地址 | 账号密码 | +|------|------|---------| +| Nacos | http://192.168.1.101:8848/nacos | nacos/nacos | +| Sentinel | http://192.168.1.101:8880 | sentinel/sentinel | +| MinIO Console | http://192.168.1.101:9101 | minioadmin/minioadmin123 | +| API Gateway | http://192.168.1.101:9000 | - | + +### 4.3 API 测试 + +```bash +# 获取验证码 +curl http://192.168.1.101:9000/auth/captcha + +# 健康检查 +curl http://192.168.1.101:9000/actuator/health +curl http://192.168.1.102:9000/actuator/health +``` + +--- + +## 五、高可用测试 + +### 5.1 测试服务故障转移 + +```bash +# 停止 Node-01 的网关 +docker stop video-gateway + +# 测试 API (应该自动切换到 Node-02) +curl http://192.168.1.102:9000/auth/captcha + +# 恢复服务 +docker start video-gateway +``` + +### 5.2 测试 Redis 主从切换 + +```bash +# 停止 Node-01 的 Redis +docker stop redis + +# 查看 Sentinel 日志 (自动将从节点提升为主) +docker logs redis-sentinel + +# 验证新的主节点 +docker exec redis redis-cli -a redis_HA_Cluster_2024 INFO replication + +# 恢复 Redis (会自动变为从节点) +docker start redis +``` + +### 5.3 测试 Nacos 集群 + +```bash +# 停止 Node-01 的 Nacos +docker stop nacos + +# 验证服务注册仍然可用 +curl http://192.168.1.102:8848/nacos/v1/ns/service/list + +# 恢复 Nacos +docker start nacos +``` + +--- + +## 六、常见问题 + +### Q1: Docker 镜像拉取失败 + +```bash +# 配置国内镜像源 +vim /etc/docker/daemon.json +# 添加镜像加速器 + +# 重启 Docker +systemctl restart docker +``` + +### Q2: Nacos 集群无法启动 + +```bash +# 检查时间同步 +apt-get install -y ntpdate +ntpdate pool.ntp.org + +# 检查数据库连接 +docker exec mysql mysql -uroot -proot123456 -e "SHOW DATABASES;" +``` + +### Q3: 内存不足 + +```bash +# 查看内存使用 +free -h + +# 调整 JVM 参数 +# 编辑 config/.env.node-xx 文件中的 JAVA_OPTS +JAVA_OPTS=-Xms128m -Xmx256m +``` + +### Q4: 端口冲突 + +```bash +# 查看端口占用 +ss -tlnp | grep -E '3306|6379|8848|9000|9002' + +# 停止冲突服务 +systemctl stop mysql +systemctl stop redis +``` + +### Q5: 微服务报 "user not found!" 错误(Nacos 认证失败) + +**问题描述**:微服务启动时报错 `com.alibaba.nacos.api.exception.NacosException: user not found!` + +**原因**:Spring Boot 3.x 默认不加载 `bootstrap.yml` 文件,导致 Nacos 认证配置未生效。 + +**解决方案**:已在 docker-compose 中添加 Spring 环境变量: + +```yaml +environment: + - SPRING_CLOUD_NACOS_DISCOVERY_USERNAME=nacos + - SPRING_CLOUD_NACOS_DISCOVERY_PASSWORD=nacos + - SPRING_CLOUD_NACOS_CONFIG_USERNAME=nacos + - SPRING_CLOUD_NACOS_CONFIG_PASSWORD=nacos +``` + +**验证**:启动后日志应显示 `username='nacos'` 而不是 `username=''` + +### Q6: MySQL 从节点初始化失败 + +**问题描述**:MySQL 从节点容器启动报错 `The MySQL server is running with the --super-read-only option` + +**原因**:MySQL 从节点配置了 `read_only=ON`,阻止了容器初始化脚本执行(创建数据库、用户等)。 + +**解决方案**: +1. 确保 `config/mysql/my-slave.cnf` 中已注释掉 `read_only` 和 `super_read_only` +2. 如果已经启动失败,需要删除数据卷重新初始化: + +```bash +# 停止容器 +docker compose -f docker-compose-node-02.yml down + +# 删除 MySQL 数据卷 +docker volume rm ha-ovp_mysql_data + +# 重新启动 +docker compose -f docker-compose-node-02.yml up -d +``` + +3. 主从复制配置完成后,手动启用只读模式: + +```bash +docker exec mysql mysql -uroot -proot123456 -e "SET GLOBAL read_only = ON; SET GLOBAL super_read_only = ON;" +``` + +#### 主从复制异常修复(复制元数据损坏) + +如果从节点执行 `START SLAVE` 或 `START REPLICA` 时报错类似: + +```text +Replica failed to initialize applier metadata structure from the repository +``` + +说明复制元数据已损坏,需要重建从库复制配置。 + +```bash +# 在 Node-02 执行 +docker exec -i mysql mysql -uroot -proot123456 << 'EOF' +STOP REPLICA; +RESET REPLICA ALL; +CHANGE REPLICATION SOURCE TO + SOURCE_HOST='192.168.1.101', + SOURCE_PORT=3306, + SOURCE_USER='repl', + SOURCE_PASSWORD='repl_password_2024', + SOURCE_AUTO_POSITION=1, + GET_SOURCE_PUBLIC_KEY=1; +START REPLICA; +SHOW REPLICA STATUS\G; +EOF + +# 确认以下字段为正常状态 +# Replica_IO_Running: Yes +# Replica_SQL_Running: Yes +# Seconds_Behind_Source: 0 + +# 恢复从库只读 +docker exec mysql mysql -uroot -proot123456 -e "SET GLOBAL read_only = ON; SET GLOBAL super_read_only = ON;" +``` + +Node-03 修复方式相同,只需将命令执行节点换为 Node-03。 + +### Q7: RocketMQ NameServer 健康检查失败 + +**问题描述**:RocketMQ NameServer 容器显示 unhealthy + +**原因**:容器内缺少 `nc` (netcat) 命令 + +**解决方案**:已修改健康检查命令使用 `pgrep`: + +```yaml +healthcheck: + test: ["CMD-SHELL", "pgrep -f 'NamesrvStartup' || exit 1"] +``` + +### Q8: Redis Sentinel 报 "Read-only file system" 错误 + +**问题描述**:Redis Sentinel 无法写入配置文件 + +**解决方案**:已修改为将配置文件复制到可写的数据卷: + +```yaml +volumes: + - sentinel_data:/data + - ./config/redis/sentinel.conf:/usr/local/etc/redis/sentinel.conf +command: sh -c "cp /usr/local/etc/redis/sentinel.conf /data/sentinel.conf && redis-sentinel /data/sentinel.conf" +``` + +### Q9: Redis 日志文件路径错误 + +**问题描述**:Redis 启动报错 `Can't open the log file` + +**解决方案**:修改 `config/redis/redis-master.conf` 和 `redis-slave.conf`: + +```conf +# 将 logfile /var/log/redis/redis-server.log 改为 +logfile "" +``` + +### Q10: Nacos 集群模式报 "dataSource or tableName is null" + +**问题描述**:Nacos 集群模式启动失败 + +**解决方案**:确保 docker-compose 中添加了 `SPRING_DATASOURCE_PLATFORM=mysql` 环境变量: + +```yaml +environment: + - MODE=cluster + - SPRING_DATASOURCE_PLATFORM=mysql + - MYSQL_SERVICE_HOST=mysql + - MYSQL_SERVICE_PORT=3306 + - MYSQL_SERVICE_DB_NAME=nacos_config +``` + +### Q11: RocketMQ Broker 启动失败报 NullPointerException + +**问题描述**:RocketMQ Broker 启动时报错: +``` +java.lang.NullPointerException + at org.apache.rocketmq.broker.schedule.ScheduleMessageService.configFilePath +``` + +**原因**: +1. Docker 命名卷权限问题:官方镜像以 `rocketmq` 用户 (UID 3000) 运行,但命名卷默认属于 root +2. 存储路径配置问题:`delayOffsetStorePath` 等路径指向不存在的子目录 + +**解决方案**: + +1. 在 docker-compose 中添加 `user: root`: +```yaml +rmqbroker: + image: apache/rocketmq:5.1.4 + user: root # 添加此行 + container_name: rmqbroker +``` + +2. 清理旧数据并重启: +```bash +docker compose -f docker-compose-node-02.yml rm -f -s rmqbroker +docker volume rm ha-ovp_rmqbroker_store ha-ovp_rmqbroker_logs +docker compose -f docker-compose-node-02.yml up -d rmqbroker +``` + +### Q12: Nacos 服务注册显示 Docker 内部 IP(假高可用问题) + +**问题描述**: +- Nacos 控制台显示服务实例 IP 为 `172.18.0.x` 等 Docker 内部 IP +- 同一服务在不同节点只显示 1 个实例(IP 冲突覆盖) +- 跨节点服务调用失败 + +**原因**:Docker 桥接网络中,每个容器获得内部虚拟 IP。微服务使用容器 IP 注册到 Nacos,导致: +1. 不同节点的容器可能获得相同 IP,Nacos 认为是同一实例 +2. Gateway 无法通过容器内部 IP 访问其他节点的服务 + +**解决方案**:在 docker-compose 中添加 `SPRING_CLOUD_NACOS_DISCOVERY_IP` 环境变量: + +```yaml +# Node-01 (192.168.1.101) +video-gateway: + environment: + - SPRING_CLOUD_NACOS_DISCOVERY_IP=${HOST_IP:-192.168.1.101} + +# Node-02 (192.168.1.102) +video-gateway: + environment: + - SPRING_CLOUD_NACOS_DISCOVERY_IP=${HOST_IP:-192.168.1.102} +``` + +**验证**:重启微服务后,Nacos 控制台应显示真实宿主机 IP(如 `192.168.1.101:9000`、`192.168.1.102:9000`) + +### Q13: Sentinel 控制台显示容器内部 IP 或实例失联 + +**问题描述**: +- Sentinel 控制台显示机器列表 IP 为 `172.18.0.x` 等 Docker 内部 IP +- 部分实例显示"失联"状态,健康实例数不正确 +- 跨节点无法进行限流降级 + +**原因**:与 Nacos 问题类似,Sentinel 客户端使用容器内部 IP 注册到 Dashboard,导致跨节点无法通信。 + +**解决方案**:在 docker-compose 中添加 `SPRING_CLOUD_SENTINEL_TRANSPORT_CLIENT_IP` 环境变量: + +```yaml +# Node-01 (192.168.1.101) +video-gateway: + environment: + - SPRING_CLOUD_SENTINEL_TRANSPORT_CLIENT_IP=${HOST_IP:-192.168.1.101} + +# Node-02 (192.168.1.102) +video-gateway: + environment: + - SPRING_CLOUD_SENTINEL_TRANSPORT_CLIENT_IP=${HOST_IP:-192.168.1.102} +``` + +**注意**:环境变量名必须是 `SPRING_CLOUD_SENTINEL_TRANSPORT_CLIENT_IP`,对应 Spring 属性 `spring.cloud.sentinel.transport.client-ip`。 + +**重要**:还需要配置以下两项: + +1. **端口映射**:每个微服务需要映射 Sentinel 通信端口(默认从 8719 开始) +2. **端口指定**:通过 `SPRING_CLOUD_SENTINEL_TRANSPORT_PORT` 指定固定端口 + +```yaml +# Node-01 微服务 Sentinel 端口分配 +video-gateway: + ports: + - "9000:9000" + - "8719:8719" # Sentinel 端口 + environment: + - SPRING_CLOUD_SENTINEL_TRANSPORT_PORT=8719 + +video-auth: + ports: + - "9001:9001" + - "8720:8720" # Sentinel 端口 + environment: + - SPRING_CLOUD_SENTINEL_TRANSPORT_PORT=8720 + +# 以此类推: user=8721, content=8722, comment=8723, recommend=8724, admin=8725 + +# Node-02 微服务使用不同端口避免冲突: 8739-8745 +``` + +3. **Dashboard 统一地址**:Node-02 及其他节点的微服务必须连接 Node-01 的 Sentinel Dashboard: +```yaml +# Node-02 微服务 +environment: + - SENTINEL_DASHBOARD_HOST=192.168.1.101 # 指向 Node-01 + - SENTINEL_DASHBOARD_PORT=8880 # Dashboard 宿主机端口 +``` + +**验证**:重启微服务后,Sentinel 控制台应显示真实宿主机 IP,且所有实例状态为"健康" + +### Q14: 微服务报 Redis READONLY 错误 + +**问题描述**:微服务启动时报错: +``` +io.lettuce.core.RedisReadOnlyException: READONLY You can't write against a read only replica. +``` + +**原因**:微服务使用单机模式连接 Redis(`REDIS_HOST`/`REDIS_PORT`),直接连接到了 Redis 从节点,而从节点是只读的。 + +**解决方案**: + +1. 使用 Nacos 配置中心管理 Redis 配置,通过 `SPRING_PROFILES_ACTIVE` 区分环境: + - `dev`: 加载 `common-redis-dev.yaml`(单机模式) + - `ha`: 加载 `common-redis-ha.yaml`(Sentinel模式) + +2. 确保 docker-compose 中配置了正确的环境变量: +```yaml +environment: + - SPRING_PROFILES_ACTIVE=ha + - REDIS_PASSWORD=${REDIS_PASSWORD:-redis_HA_Cluster_2024} + - REDIS_SENTINEL_NODES=${REDIS_SENTINEL_NODES:-192.168.1.101:26379,192.168.1.102:26379,192.168.1.103:26379} +``` + +3. 确保 Nacos 中 `common-redis-ha.yaml` 配置正确: +```yaml +spring: + data: + redis: + password: ${REDIS_PASSWORD:redis_HA_Cluster_2024} + sentinel: + master: mymaster + nodes: ${REDIS_SENTINEL_NODES:192.168.1.101:26379,192.168.1.102:26379,192.168.1.103:26379} + password: ${REDIS_PASSWORD:redis_HA_Cluster_2024} +``` + +**验证**: +```bash +# 查看当前 Redis 主节点 +docker exec redis-sentinel redis-cli -p 26379 SENTINEL get-master-addr-by-name mymaster +``` + +### Q15: Nacos 配置不生效或配置冲突 + +**问题描述**:修改 Nacos 配置后服务没有更新,或启动时配置异常。 + +**原因**: +1. 本地 `application.yml` 与 Nacos 配置冲突 +2. 配置加载顺序问题 +3. `spring.config.import` 配置错误 + +**解决方案**: + +1. **确保本地 application.yml 极简**:只保留 Nacos 连接配置 +```yaml +spring: + application: + name: video-auth + profiles: + active: ${SPRING_PROFILES_ACTIVE:dev} + config: + import: + - optional:nacos:common.yaml?refreshEnabled=true + - optional:nacos:common-redis-${spring.profiles.active}.yaml?refreshEnabled=true + - optional:nacos:${spring.application.name}.yaml?refreshEnabled=true + cloud: + nacos: + discovery: + server-addr: ${NACOS_SERVER_ADDR:127.0.0.1:8848} + namespace: ${NACOS_NAMESPACE:} + username: ${NACOS_USERNAME:nacos} + password: ${NACOS_PASSWORD:nacos} +``` + +2. **配置加载顺序**: + - common.yaml(公共配置) + - common-redis-${profile}.yaml(环境配置) + - ${spring.application.name}.yaml(服务配置) + +3. **验证配置加载**: +```bash +# 查看服务启动日志 +docker logs video-auth 2>&1 | grep -i nacos +# 应看到类似: "Loading nacos config, dataId: common.yaml" +``` + +4. **强制刷新配置**:修改 Nacos 配置后,服务会自动感知更新(`refreshEnabled=true`) + +### Q16: 头像或封面上传成功但访问 404 + +**问题描述**: +- 用户头像上传接口返回成功 +- 但浏览器访问图片地址时返回 `404 Not Found` +- 常见于头像、视频封面等图片资源 + +**原因**: +- 对象被上传到了 `Node-02 MinIO` +- 前端固定从 `Node-01 MinIO` 读取 +- 由于两个 MinIO 实例不是分布式共享存储,导致对象只存在于上传节点 + +**解决方案**: + +1. 统一所有业务上传写入 `Node-01 MinIO`: +```yaml +MINIO_HOST=192.168.1.101 +MINIO_PORT=9100 +``` + +2. 在 `Node-01` 执行同步脚本,把对象同步到 `Node-02`: +```bash +cd /opt/HA-OVP/backend +bash scripts/minio-sync-node01-to-node02.sh +``` + +3. 配置 cron 定时同步,避免后续对象再次不一致: +```bash +*/5 * * * * /bin/bash /opt/HA-OVP/backend/scripts/minio-sync-node01-to-node02.sh >> /var/log/minio-sync.log 2>&1 +``` + +**说明**: +- 该方案是“统一写 Node-01 + 同步到 Node-02”的稳定方案 +- 不要将当前架构表述为真正的 MinIO 分布式集群 + +--- + +## 七、日常运维 + +### 7.1 日志查看 + +```bash +# 查看服务日志 +docker logs -f --tail 100 video-gateway +docker logs -f --tail 100 video-auth + +# 查看所有服务状态 +docker compose -f docker-compose-node-01.yml ps +``` + +### 7.2 服务重启 + +```bash +# 重启单个服务 +docker restart video-gateway + +# 重启所有服务 +docker compose -f docker-compose-node-01.yml restart +``` + +### 7.3 数据备份 + +```bash +# MySQL 备份 +docker exec mysql mysqldump -uroot -proot123456 --all-databases > backup_$(date +%Y%m%d).sql + +# Redis 备份 +docker exec redis redis-cli -a redis_HA_Cluster_2024 BGSAVE +docker cp redis:/data/dump.rdb redis_backup_$(date +%Y%m%d).rdb + +# MinIO 备份 +mc mirror myminio/video-platform minio_backup_$(date +%Y%m%d) +``` + +--- + +## 八、性能调优 + +### 8.1 JVM 调优 + +根据实际内存调整 JVM 参数: + +```bash +# 8GB 内存服务器建议配置 +JAVA_OPTS=-Xms512m -Xmx1024m -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=256m -XX:+UseG1GC + +# 16GB 内存服务器建议配置 +JAVA_OPTS=-Xms1g -Xmx2g -XX:MetaspaceSize=256m -XX:MaxMetaspaceSize=512m -XX:+UseG1GC +``` + +### 8.2 MySQL 调优 + +```bash +# 编辑 MySQL 配置 +vim config/mysql/my-master.cnf + +# 关键参数 +innodb_buffer_pool_size = 2G # 设置为物理内存的 50-70% +innodb_log_file_size = 512M +max_connections = 500 +``` + +### 8.3 Redis 调优 + +```bash +# 编辑 Redis 配置 +vim config/redis/redis-master.conf + +# 关键参数 +maxmemory 2gb +maxmemory-policy allkeys-lru +``` + +--- + +## 九、架构图 + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ 高可用集群架构 │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ │ +│ ┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ +│ │ Node-01 │ │ Node-02 │ │ Node-03 │ +│ │ 192.168.1.101 │ │ 192.168.1.102 │ │ 192.168.1.103 │ +│ ├─────────────────┤ ├─────────────────┤ ├─────────────────┤ +│ │ MySQL Master │ ──────▶ │ MySQL Slave │ ──────▶ │ MySQL Slave │ +│ │ Redis Master │ ◀────── │ Redis Slave │ ◀────── │ Redis Slave │ +│ │ Nacos Node1 │ ◀─────▶ │ Nacos Node2 │ ◀─────▶ │ Nacos Node3 │ +│ │ NameServer │ │ NameServer │ │ NameServer │ +│ │ MinIO :9100 │ │ Broker-A :10911 │ │ Broker-B :10911 │ +│ │ Sentinel :8880 │ │ MinIO :9100 │ │ MinIO :9000 │ +│ ├─────────────────┤ │ Sentinel :8880 │ │ Sentinel :8880 │ +│ │ Gateway :9000 │ ├─────────────────┤ └─────────────────┘ +│ │ Auth :9001 │ │ Gateway :9000 │ (备份节点) +│ │ User :9002 │ │ Auth :9001 │ +│ │ Content :9003 │ │ User :9002 │ +│ │ Comment :9004 │ │ Content :9003 │ +│ │ Recommend :9005 │ │ Comment :9004 │ +│ │ Admin :9006 │ │ Recommend :9005 │ +│ │ Nginx :80 │ │ Admin :9006 │ +│ └─────────────────┘ └─────────────────┘ +│ (主节点,统一对象写入点) (从节点,对象同步副本) +└─────────────────────────────────────────────────────────────────────────────┘ +```