feat(架构): 完善基础架构设计
This commit is contained in:
389
internal/shared/resilience/circuit_breaker.go
Normal file
389
internal/shared/resilience/circuit_breaker.go
Normal file
@@ -0,0 +1,389 @@
|
||||
package resilience
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// CircuitState 熔断器状态
|
||||
type CircuitState int
|
||||
|
||||
const (
|
||||
// StateClosed 关闭状态（正常）
|
||||
StateClosed CircuitState = iota
|
||||
// StateOpen 开启状态（熔断）
|
||||
StateOpen
|
||||
// StateHalfOpen 半开状态（测试）
|
||||
StateHalfOpen
|
||||
)
|
||||
|
||||
func (s CircuitState) String() string {
|
||||
switch s {
|
||||
case StateClosed:
|
||||
return "CLOSED"
|
||||
case StateOpen:
|
||||
return "OPEN"
|
||||
case StateHalfOpen:
|
||||
return "HALF_OPEN"
|
||||
default:
|
||||
return "UNKNOWN"
|
||||
}
|
||||
}
|
||||
|
||||
// CircuitBreakerConfig 熔断器配置
|
||||
type CircuitBreakerConfig struct {
|
||||
// 故障阈值
|
||||
FailureThreshold int
|
||||
// 重置超时时间
|
||||
ResetTimeout time.Duration
|
||||
// 检测窗口大小
|
||||
WindowSize int
|
||||
// 半开状态允许的请求数
|
||||
HalfOpenMaxRequests int
|
||||
// 成功阈值（半开->关闭）
|
||||
SuccessThreshold int
|
||||
}
|
||||
|
||||
// DefaultCircuitBreakerConfig 默认熔断器配置
|
||||
func DefaultCircuitBreakerConfig() CircuitBreakerConfig {
|
||||
return CircuitBreakerConfig{
|
||||
FailureThreshold: 5,
|
||||
ResetTimeout: 60 * time.Second,
|
||||
WindowSize: 10,
|
||||
HalfOpenMaxRequests: 3,
|
||||
SuccessThreshold: 2,
|
||||
}
|
||||
}
|
||||
|
||||
// CircuitBreaker 熔断器
|
||||
type CircuitBreaker struct {
|
||||
config CircuitBreakerConfig
|
||||
logger *zap.Logger
|
||||
mutex sync.RWMutex
|
||||
|
||||
// 状态
|
||||
state CircuitState
|
||||
|
||||
// 计数器
|
||||
failures int
|
||||
successes int
|
||||
requests int
|
||||
consecutiveFailures int
|
||||
|
||||
// 时间记录
|
||||
lastFailTime time.Time
|
||||
lastStateChange time.Time
|
||||
|
||||
// 统计窗口
|
||||
window []bool // true=success, false=failure
|
||||
windowIndex int
|
||||
windowFull bool
|
||||
|
||||
// 事件回调
|
||||
onStateChange func(from, to CircuitState)
|
||||
}
|
||||
|
||||
// NewCircuitBreaker 创建熔断器
|
||||
func NewCircuitBreaker(config CircuitBreakerConfig, logger *zap.Logger) *CircuitBreaker {
|
||||
cb := &CircuitBreaker{
|
||||
config: config,
|
||||
logger: logger,
|
||||
state: StateClosed,
|
||||
window: make([]bool, config.WindowSize),
|
||||
lastStateChange: time.Now(),
|
||||
}
|
||||
|
||||
return cb
|
||||
}
|
||||
|
||||
// Execute 执行函数，如果熔断器开启则快速失败
|
||||
func (cb *CircuitBreaker) Execute(ctx context.Context, fn func() error) error {
|
||||
// 检查是否允许执行
|
||||
if !cb.allowRequest() {
|
||||
return ErrCircuitBreakerOpen
|
||||
}
|
||||
|
||||
// 执行函数
|
||||
start := time.Now()
|
||||
err := fn()
|
||||
duration := time.Since(start)
|
||||
|
||||
// 记录结果
|
||||
cb.recordResult(err == nil, duration)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// allowRequest 检查是否允许请求
|
||||
func (cb *CircuitBreaker) allowRequest() bool {
|
||||
cb.mutex.Lock()
|
||||
defer cb.mutex.Unlock()
|
||||
|
||||
now := time.Now()
|
||||
|
||||
switch cb.state {
|
||||
case StateClosed:
|
||||
return true
|
||||
|
||||
case StateOpen:
|
||||
// 检查是否到了重置时间
|
||||
if now.Sub(cb.lastStateChange) > cb.config.ResetTimeout {
|
||||
cb.setState(StateHalfOpen)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
|
||||
case StateHalfOpen:
|
||||
// 半开状态下限制请求数
|
||||
return cb.requests < cb.config.HalfOpenMaxRequests
|
||||
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// recordResult 记录执行结果
|
||||
func (cb *CircuitBreaker) recordResult(success bool, duration time.Duration) {
|
||||
cb.mutex.Lock()
|
||||
defer cb.mutex.Unlock()
|
||||
|
||||
cb.requests++
|
||||
|
||||
// 更新滑动窗口
|
||||
cb.updateWindow(success)
|
||||
|
||||
if success {
|
||||
cb.successes++
|
||||
cb.consecutiveFailures = 0
|
||||
cb.onSuccess()
|
||||
} else {
|
||||
cb.failures++
|
||||
cb.consecutiveFailures++
|
||||
cb.lastFailTime = time.Now()
|
||||
cb.onFailure()
|
||||
}
|
||||
|
||||
cb.logger.Debug("Circuit breaker recorded result",
|
||||
zap.Bool("success", success),
|
||||
zap.Duration("duration", duration),
|
||||
zap.String("state", cb.state.String()),
|
||||
zap.Int("failures", cb.failures),
|
||||
zap.Int("successes", cb.successes))
|
||||
}
|
||||
|
||||
// updateWindow 更新滑动窗口
|
||||
func (cb *CircuitBreaker) updateWindow(success bool) {
|
||||
cb.window[cb.windowIndex] = success
|
||||
cb.windowIndex = (cb.windowIndex + 1) % cb.config.WindowSize
|
||||
|
||||
if cb.windowIndex == 0 {
|
||||
cb.windowFull = true
|
||||
}
|
||||
}
|
||||
|
||||
// onSuccess 成功时的处理
|
||||
func (cb *CircuitBreaker) onSuccess() {
|
||||
if cb.state == StateHalfOpen {
|
||||
// 半开状态下，如果成功次数达到阈值，则关闭熔断器
|
||||
if cb.successes >= cb.config.SuccessThreshold {
|
||||
cb.setState(StateClosed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// onFailure 失败时的处理
|
||||
func (cb *CircuitBreaker) onFailure() {
|
||||
if cb.state == StateClosed {
|
||||
// 关闭状态下，检查是否需要开启熔断器
|
||||
if cb.shouldTrip() {
|
||||
cb.setState(StateOpen)
|
||||
}
|
||||
} else if cb.state == StateHalfOpen {
|
||||
// 半开状态下，如果失败则立即开启熔断器
|
||||
cb.setState(StateOpen)
|
||||
}
|
||||
}
|
||||
|
||||
// shouldTrip 检查是否应该触发熔断
|
||||
func (cb *CircuitBreaker) shouldTrip() bool {
|
||||
// 基于连续失败次数
|
||||
if cb.consecutiveFailures >= cb.config.FailureThreshold {
|
||||
return true
|
||||
}
|
||||
|
||||
// 基于滑动窗口的失败率
|
||||
if cb.windowFull {
|
||||
failures := 0
|
||||
for _, success := range cb.window {
|
||||
if !success {
|
||||
failures++
|
||||
}
|
||||
}
|
||||
|
||||
failureRate := float64(failures) / float64(cb.config.WindowSize)
|
||||
return failureRate >= 0.5 // 50%失败率
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// setState 设置状态
|
||||
func (cb *CircuitBreaker) setState(newState CircuitState) {
|
||||
if cb.state == newState {
|
||||
return
|
||||
}
|
||||
|
||||
oldState := cb.state
|
||||
cb.state = newState
|
||||
cb.lastStateChange = time.Now()
|
||||
|
||||
// 重置计数器
|
||||
if newState == StateClosed {
|
||||
cb.requests = 0
|
||||
cb.failures = 0
|
||||
cb.successes = 0
|
||||
cb.consecutiveFailures = 0
|
||||
} else if newState == StateHalfOpen {
|
||||
cb.requests = 0
|
||||
cb.successes = 0
|
||||
}
|
||||
|
||||
cb.logger.Info("Circuit breaker state changed",
|
||||
zap.String("from", oldState.String()),
|
||||
zap.String("to", newState.String()),
|
||||
zap.Int("failures", cb.failures),
|
||||
zap.Int("successes", cb.successes))
|
||||
|
||||
// 触发状态变更回调
|
||||
if cb.onStateChange != nil {
|
||||
cb.onStateChange(oldState, newState)
|
||||
}
|
||||
}
|
||||
|
||||
// GetState 获取当前状态
|
||||
func (cb *CircuitBreaker) GetState() CircuitState {
|
||||
cb.mutex.RLock()
|
||||
defer cb.mutex.RUnlock()
|
||||
return cb.state
|
||||
}
|
||||
|
||||
// GetStats 获取统计信息
|
||||
func (cb *CircuitBreaker) GetStats() CircuitBreakerStats {
|
||||
cb.mutex.RLock()
|
||||
defer cb.mutex.RUnlock()
|
||||
|
||||
return CircuitBreakerStats{
|
||||
State: cb.state.String(),
|
||||
Failures: cb.failures,
|
||||
Successes: cb.successes,
|
||||
Requests: cb.requests,
|
||||
ConsecutiveFailures: cb.consecutiveFailures,
|
||||
LastFailTime: cb.lastFailTime,
|
||||
LastStateChange: cb.lastStateChange,
|
||||
FailureThreshold: cb.config.FailureThreshold,
|
||||
ResetTimeout: cb.config.ResetTimeout,
|
||||
}
|
||||
}
|
||||
|
||||
// Reset 重置熔断器
|
||||
func (cb *CircuitBreaker) Reset() {
|
||||
cb.mutex.Lock()
|
||||
defer cb.mutex.Unlock()
|
||||
|
||||
cb.setState(StateClosed)
|
||||
cb.window = make([]bool, cb.config.WindowSize)
|
||||
cb.windowIndex = 0
|
||||
cb.windowFull = false
|
||||
|
||||
cb.logger.Info("Circuit breaker reset")
|
||||
}
|
||||
|
||||
// SetStateChangeCallback 设置状态变更回调
|
||||
func (cb *CircuitBreaker) SetStateChangeCallback(callback func(from, to CircuitState)) {
|
||||
cb.mutex.Lock()
|
||||
defer cb.mutex.Unlock()
|
||||
cb.onStateChange = callback
|
||||
}
|
||||
|
||||
// CircuitBreakerStats 熔断器统计信息
|
||||
type CircuitBreakerStats struct {
|
||||
State string `json:"state"`
|
||||
Failures int `json:"failures"`
|
||||
Successes int `json:"successes"`
|
||||
Requests int `json:"requests"`
|
||||
ConsecutiveFailures int `json:"consecutive_failures"`
|
||||
LastFailTime time.Time `json:"last_fail_time"`
|
||||
LastStateChange time.Time `json:"last_state_change"`
|
||||
FailureThreshold int `json:"failure_threshold"`
|
||||
ResetTimeout time.Duration `json:"reset_timeout"`
|
||||
}
|
||||
|
||||
// 预定义错误
|
||||
var (
|
||||
ErrCircuitBreakerOpen = errors.New("circuit breaker is open")
|
||||
)
|
||||
|
||||
// Wrapper 熔断器包装器
|
||||
type Wrapper struct {
|
||||
breakers map[string]*CircuitBreaker
|
||||
logger *zap.Logger
|
||||
mutex sync.RWMutex
|
||||
}
|
||||
|
||||
// NewWrapper 创建熔断器包装器
|
||||
func NewWrapper(logger *zap.Logger) *Wrapper {
|
||||
return &Wrapper{
|
||||
breakers: make(map[string]*CircuitBreaker),
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
// GetOrCreate 获取或创建熔断器
|
||||
func (w *Wrapper) GetOrCreate(name string, config CircuitBreakerConfig) *CircuitBreaker {
|
||||
w.mutex.Lock()
|
||||
defer w.mutex.Unlock()
|
||||
|
||||
if cb, exists := w.breakers[name]; exists {
|
||||
return cb
|
||||
}
|
||||
|
||||
cb := NewCircuitBreaker(config, w.logger.Named(name))
|
||||
w.breakers[name] = cb
|
||||
|
||||
w.logger.Info("Created circuit breaker", zap.String("name", name))
|
||||
return cb
|
||||
}
|
||||
|
||||
// Execute 执行带熔断器的函数
|
||||
func (w *Wrapper) Execute(ctx context.Context, name string, fn func() error) error {
|
||||
cb := w.GetOrCreate(name, DefaultCircuitBreakerConfig())
|
||||
return cb.Execute(ctx, fn)
|
||||
}
|
||||
|
||||
// GetStats 获取所有熔断器统计
|
||||
func (w *Wrapper) GetStats() map[string]CircuitBreakerStats {
|
||||
w.mutex.RLock()
|
||||
defer w.mutex.RUnlock()
|
||||
|
||||
stats := make(map[string]CircuitBreakerStats)
|
||||
for name, cb := range w.breakers {
|
||||
stats[name] = cb.GetStats()
|
||||
}
|
||||
|
||||
return stats
|
||||
}
|
||||
|
||||
// ResetAll 重置所有熔断器
|
||||
func (w *Wrapper) ResetAll() {
|
||||
w.mutex.RLock()
|
||||
defer w.mutex.RUnlock()
|
||||
|
||||
for name, cb := range w.breakers {
|
||||
cb.Reset()
|
||||
w.logger.Info("Reset circuit breaker", zap.String("name", name))
|
||||
}
|
||||
}
|
||||
467
internal/shared/resilience/retry.go
Normal file
467
internal/shared/resilience/retry.go
Normal file
@@ -0,0 +1,467 @@
|
||||
package resilience
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// RetryConfig 重试配置
|
||||
type RetryConfig struct {
|
||||
// 最大重试次数
|
||||
MaxAttempts int
|
||||
// 初始延迟
|
||||
InitialDelay time.Duration
|
||||
// 最大延迟
|
||||
MaxDelay time.Duration
|
||||
// 退避倍数
|
||||
BackoffMultiplier float64
|
||||
// 抖动系数
|
||||
JitterFactor float64
|
||||
// 重试条件
|
||||
RetryCondition func(error) bool
|
||||
// 延迟函数
|
||||
DelayFunc func(attempt int, config RetryConfig) time.Duration
|
||||
}
|
||||
|
||||
// DefaultRetryConfig 默认重试配置
|
||||
func DefaultRetryConfig() RetryConfig {
|
||||
return RetryConfig{
|
||||
MaxAttempts: 3,
|
||||
InitialDelay: 100 * time.Millisecond,
|
||||
MaxDelay: 5 * time.Second,
|
||||
BackoffMultiplier: 2.0,
|
||||
JitterFactor: 0.1,
|
||||
RetryCondition: DefaultRetryCondition,
|
||||
DelayFunc: ExponentialBackoffWithJitter,
|
||||
}
|
||||
}
|
||||
|
||||
// RetryableError 可重试错误接口
|
||||
type RetryableError interface {
|
||||
error
|
||||
IsRetryable() bool
|
||||
}
|
||||
|
||||
// DefaultRetryCondition 默认重试条件
|
||||
func DefaultRetryCondition(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
// 检查是否实现了RetryableError接口
|
||||
if retryable, ok := err.(RetryableError); ok {
|
||||
return retryable.IsRetryable()
|
||||
}
|
||||
|
||||
// 默认所有错误都重试
|
||||
return true
|
||||
}
|
||||
|
||||
// IsRetryableHTTPError HTTP错误重试条件
|
||||
func IsRetryableHTTPError(statusCode int) bool {
|
||||
// 5xx错误通常可以重试
|
||||
// 429（Too Many Requests）也可以重试
|
||||
return statusCode >= 500 || statusCode == 429
|
||||
}
|
||||
|
||||
// DelayFunction 延迟函数类型
|
||||
type DelayFunction func(attempt int, config RetryConfig) time.Duration
|
||||
|
||||
// FixedDelay 固定延迟
|
||||
func FixedDelay(attempt int, config RetryConfig) time.Duration {
|
||||
return config.InitialDelay
|
||||
}
|
||||
|
||||
// LinearBackoff 线性退避
|
||||
func LinearBackoff(attempt int, config RetryConfig) time.Duration {
|
||||
delay := time.Duration(attempt) * config.InitialDelay
|
||||
if delay > config.MaxDelay {
|
||||
delay = config.MaxDelay
|
||||
}
|
||||
return delay
|
||||
}
|
||||
|
||||
// ExponentialBackoff 指数退避
|
||||
func ExponentialBackoff(attempt int, config RetryConfig) time.Duration {
|
||||
delay := config.InitialDelay
|
||||
for i := 0; i < attempt; i++ {
|
||||
delay = time.Duration(float64(delay) * config.BackoffMultiplier)
|
||||
}
|
||||
|
||||
if delay > config.MaxDelay {
|
||||
delay = config.MaxDelay
|
||||
}
|
||||
|
||||
return delay
|
||||
}
|
||||
|
||||
// ExponentialBackoffWithJitter 带抖动的指数退避
|
||||
func ExponentialBackoffWithJitter(attempt int, config RetryConfig) time.Duration {
|
||||
delay := ExponentialBackoff(attempt, config)
|
||||
|
||||
// 添加抖动
|
||||
jitter := config.JitterFactor
|
||||
if jitter > 0 {
|
||||
jitterRange := float64(delay) * jitter
|
||||
jitterOffset := (rand.Float64() - 0.5) * 2 * jitterRange
|
||||
delay = time.Duration(float64(delay) + jitterOffset)
|
||||
}
|
||||
|
||||
if delay < 0 {
|
||||
delay = config.InitialDelay
|
||||
}
|
||||
|
||||
return delay
|
||||
}
|
||||
|
||||
// RetryStats 重试统计
|
||||
type RetryStats struct {
|
||||
TotalAttempts int `json:"total_attempts"`
|
||||
Successes int `json:"successes"`
|
||||
Failures int `json:"failures"`
|
||||
TotalRetries int `json:"total_retries"`
|
||||
AverageAttempts float64 `json:"average_attempts"`
|
||||
TotalDelay time.Duration `json:"total_delay"`
|
||||
LastError string `json:"last_error,omitempty"`
|
||||
}
|
||||
|
||||
// Retryer 重试器
|
||||
type Retryer struct {
|
||||
config RetryConfig
|
||||
logger *zap.Logger
|
||||
stats RetryStats
|
||||
}
|
||||
|
||||
// NewRetryer 创建重试器
|
||||
func NewRetryer(config RetryConfig, logger *zap.Logger) *Retryer {
|
||||
if config.DelayFunc == nil {
|
||||
config.DelayFunc = ExponentialBackoffWithJitter
|
||||
}
|
||||
if config.RetryCondition == nil {
|
||||
config.RetryCondition = DefaultRetryCondition
|
||||
}
|
||||
|
||||
return &Retryer{
|
||||
config: config,
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
// Execute 执行带重试的函数
|
||||
func (r *Retryer) Execute(ctx context.Context, operation func() error) error {
|
||||
return r.ExecuteWithResult(ctx, func() (interface{}, error) {
|
||||
return nil, operation()
|
||||
})
|
||||
}
|
||||
|
||||
// ExecuteWithResult 执行带重试和返回值的函数
|
||||
func (r *Retryer) ExecuteWithResult(ctx context.Context, operation func() (interface{}, error)) error {
|
||||
var lastErr error
|
||||
startTime := time.Now()
|
||||
|
||||
for attempt := 0; attempt < r.config.MaxAttempts; attempt++ {
|
||||
// 检查上下文是否被取消
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
// 执行操作
|
||||
attemptStart := time.Now()
|
||||
_, err := operation()
|
||||
attemptDuration := time.Since(attemptStart)
|
||||
|
||||
// 更新统计
|
||||
r.stats.TotalAttempts++
|
||||
if err == nil {
|
||||
r.stats.Successes++
|
||||
r.logger.Debug("Operation succeeded",
|
||||
zap.Int("attempt", attempt+1),
|
||||
zap.Duration("duration", attemptDuration))
|
||||
return nil
|
||||
}
|
||||
|
||||
lastErr = err
|
||||
r.stats.Failures++
|
||||
if attempt > 0 {
|
||||
r.stats.TotalRetries++
|
||||
}
|
||||
|
||||
// 检查是否应该重试
|
||||
if !r.config.RetryCondition(err) {
|
||||
r.logger.Debug("Error is not retryable",
|
||||
zap.Error(err),
|
||||
zap.Int("attempt", attempt+1))
|
||||
break
|
||||
}
|
||||
|
||||
// 如果这是最后一次尝试，不需要延迟
|
||||
if attempt == r.config.MaxAttempts-1 {
|
||||
r.logger.Debug("Reached max attempts",
|
||||
zap.Error(err),
|
||||
zap.Int("max_attempts", r.config.MaxAttempts))
|
||||
break
|
||||
}
|
||||
|
||||
// 计算延迟
|
||||
delay := r.config.DelayFunc(attempt, r.config)
|
||||
r.stats.TotalDelay += delay
|
||||
|
||||
r.logger.Debug("Operation failed, retrying",
|
||||
zap.Error(err),
|
||||
zap.Int("attempt", attempt+1),
|
||||
zap.Duration("delay", delay),
|
||||
zap.Duration("attempt_duration", attemptDuration))
|
||||
|
||||
// 等待重试
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(delay):
|
||||
}
|
||||
}
|
||||
|
||||
// 更新最终统计
|
||||
totalDuration := time.Since(startTime)
|
||||
if r.stats.TotalAttempts > 0 {
|
||||
r.stats.AverageAttempts = float64(r.stats.TotalRetries) / float64(r.stats.Successes+r.stats.Failures)
|
||||
}
|
||||
if lastErr != nil {
|
||||
r.stats.LastError = lastErr.Error()
|
||||
}
|
||||
|
||||
r.logger.Warn("Operation failed after all retries",
|
||||
zap.Error(lastErr),
|
||||
zap.Int("total_attempts", r.stats.TotalAttempts),
|
||||
zap.Duration("total_duration", totalDuration))
|
||||
|
||||
return fmt.Errorf("operation failed after %d attempts: %w", r.config.MaxAttempts, lastErr)
|
||||
}
|
||||
|
||||
// GetStats 获取重试统计
|
||||
func (r *Retryer) GetStats() RetryStats {
|
||||
return r.stats
|
||||
}
|
||||
|
||||
// Reset 重置统计
|
||||
func (r *Retryer) Reset() {
|
||||
r.stats = RetryStats{}
|
||||
r.logger.Debug("Retry stats reset")
|
||||
}
|
||||
|
||||
// Retry 简单重试函数
|
||||
func Retry(ctx context.Context, config RetryConfig, operation func() error) error {
|
||||
retryer := NewRetryer(config, zap.NewNop())
|
||||
return retryer.Execute(ctx, operation)
|
||||
}
|
||||
|
||||
// RetryWithResult 带返回值的重试函数
|
||||
func RetryWithResult[T any](ctx context.Context, config RetryConfig, operation func() (T, error)) (T, error) {
|
||||
var result T
|
||||
var finalErr error
|
||||
|
||||
retryer := NewRetryer(config, zap.NewNop())
|
||||
err := retryer.ExecuteWithResult(ctx, func() (interface{}, error) {
|
||||
r, e := operation()
|
||||
result = r
|
||||
return r, e
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
finalErr = err
|
||||
}
|
||||
|
||||
return result, finalErr
|
||||
}
|
||||
|
||||
// 预定义的重试配置
|
||||
|
||||
// QuickRetry 快速重试（适用于轻量级操作）
|
||||
func QuickRetry() RetryConfig {
|
||||
return RetryConfig{
|
||||
MaxAttempts: 3,
|
||||
InitialDelay: 50 * time.Millisecond,
|
||||
MaxDelay: 500 * time.Millisecond,
|
||||
BackoffMultiplier: 2.0,
|
||||
JitterFactor: 0.1,
|
||||
RetryCondition: DefaultRetryCondition,
|
||||
DelayFunc: ExponentialBackoffWithJitter,
|
||||
}
|
||||
}
|
||||
|
||||
// StandardRetry 标准重试（适用于一般操作）
|
||||
func StandardRetry() RetryConfig {
|
||||
return DefaultRetryConfig()
|
||||
}
|
||||
|
||||
// PatientRetry 耐心重试（适用于重要操作）
|
||||
func PatientRetry() RetryConfig {
|
||||
return RetryConfig{
|
||||
MaxAttempts: 5,
|
||||
InitialDelay: 200 * time.Millisecond,
|
||||
MaxDelay: 10 * time.Second,
|
||||
BackoffMultiplier: 2.0,
|
||||
JitterFactor: 0.2,
|
||||
RetryCondition: DefaultRetryCondition,
|
||||
DelayFunc: ExponentialBackoffWithJitter,
|
||||
}
|
||||
}
|
||||
|
||||
// DatabaseRetry 数据库重试配置
|
||||
func DatabaseRetry() RetryConfig {
|
||||
return RetryConfig{
|
||||
MaxAttempts: 3,
|
||||
InitialDelay: 100 * time.Millisecond,
|
||||
MaxDelay: 2 * time.Second,
|
||||
BackoffMultiplier: 1.5,
|
||||
JitterFactor: 0.1,
|
||||
RetryCondition: func(err error) bool {
|
||||
// 这里可以根据具体的数据库错误类型判断
|
||||
// 例如：连接超时、临时网络错误等
|
||||
return DefaultRetryCondition(err)
|
||||
},
|
||||
DelayFunc: ExponentialBackoffWithJitter,
|
||||
}
|
||||
}
|
||||
|
||||
// HTTPRetry HTTP重试配置
|
||||
func HTTPRetry() RetryConfig {
|
||||
return RetryConfig{
|
||||
MaxAttempts: 3,
|
||||
InitialDelay: 200 * time.Millisecond,
|
||||
MaxDelay: 5 * time.Second,
|
||||
BackoffMultiplier: 2.0,
|
||||
JitterFactor: 0.15,
|
||||
RetryCondition: func(err error) bool {
|
||||
// HTTP相关的重试条件
|
||||
return DefaultRetryCondition(err)
|
||||
},
|
||||
DelayFunc: ExponentialBackoffWithJitter,
|
||||
}
|
||||
}
|
||||
|
||||
// RetryManager 重试管理器
|
||||
type RetryManager struct {
|
||||
retryers map[string]*Retryer
|
||||
logger *zap.Logger
|
||||
mutex sync.RWMutex
|
||||
}
|
||||
|
||||
// NewRetryManager 创建重试管理器
|
||||
func NewRetryManager(logger *zap.Logger) *RetryManager {
|
||||
return &RetryManager{
|
||||
retryers: make(map[string]*Retryer),
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
// GetOrCreate 获取或创建重试器
|
||||
func (rm *RetryManager) GetOrCreate(name string, config RetryConfig) *Retryer {
|
||||
rm.mutex.Lock()
|
||||
defer rm.mutex.Unlock()
|
||||
|
||||
if retryer, exists := rm.retryers[name]; exists {
|
||||
return retryer
|
||||
}
|
||||
|
||||
retryer := NewRetryer(config, rm.logger.Named(name))
|
||||
rm.retryers[name] = retryer
|
||||
|
||||
rm.logger.Info("Created retryer", zap.String("name", name))
|
||||
return retryer
|
||||
}
|
||||
|
||||
// Execute 执行带重试的操作
|
||||
func (rm *RetryManager) Execute(ctx context.Context, name string, operation func() error) error {
|
||||
retryer := rm.GetOrCreate(name, DefaultRetryConfig())
|
||||
return retryer.Execute(ctx, operation)
|
||||
}
|
||||
|
||||
// GetStats 获取所有重试器统计
|
||||
func (rm *RetryManager) GetStats() map[string]RetryStats {
|
||||
rm.mutex.RLock()
|
||||
defer rm.mutex.RUnlock()
|
||||
|
||||
stats := make(map[string]RetryStats)
|
||||
for name, retryer := range rm.retryers {
|
||||
stats[name] = retryer.GetStats()
|
||||
}
|
||||
|
||||
return stats
|
||||
}
|
||||
|
||||
// ResetAll 重置所有重试器统计
|
||||
func (rm *RetryManager) ResetAll() {
|
||||
rm.mutex.RLock()
|
||||
defer rm.mutex.RUnlock()
|
||||
|
||||
for name, retryer := range rm.retryers {
|
||||
retryer.Reset()
|
||||
rm.logger.Info("Reset retryer stats", zap.String("name", name))
|
||||
}
|
||||
}
|
||||
|
||||
// RetryerWrapper 重试器包装器
|
||||
type RetryerWrapper struct {
|
||||
manager *RetryManager
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
// NewRetryerWrapper 创建重试器包装器
|
||||
func NewRetryerWrapper(logger *zap.Logger) *RetryerWrapper {
|
||||
return &RetryerWrapper{
|
||||
manager: NewRetryManager(logger),
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
// ExecuteWithQuickRetry 执行快速重试
|
||||
func (rw *RetryerWrapper) ExecuteWithQuickRetry(ctx context.Context, name string, operation func() error) error {
|
||||
retryer := rw.manager.GetOrCreate(name+".quick", QuickRetry())
|
||||
return retryer.Execute(ctx, operation)
|
||||
}
|
||||
|
||||
// ExecuteWithStandardRetry 执行标准重试
|
||||
func (rw *RetryerWrapper) ExecuteWithStandardRetry(ctx context.Context, name string, operation func() error) error {
|
||||
retryer := rw.manager.GetOrCreate(name+".standard", StandardRetry())
|
||||
return retryer.Execute(ctx, operation)
|
||||
}
|
||||
|
||||
// ExecuteWithDatabaseRetry 执行数据库重试
|
||||
func (rw *RetryerWrapper) ExecuteWithDatabaseRetry(ctx context.Context, name string, operation func() error) error {
|
||||
retryer := rw.manager.GetOrCreate(name+".database", DatabaseRetry())
|
||||
return retryer.Execute(ctx, operation)
|
||||
}
|
||||
|
||||
// ExecuteWithHTTPRetry 执行HTTP重试
|
||||
func (rw *RetryerWrapper) ExecuteWithHTTPRetry(ctx context.Context, name string, operation func() error) error {
|
||||
retryer := rw.manager.GetOrCreate(name+".http", HTTPRetry())
|
||||
return retryer.Execute(ctx, operation)
|
||||
}
|
||||
|
||||
// ExecuteWithCustomRetry 执行自定义重试
|
||||
func (rw *RetryerWrapper) ExecuteWithCustomRetry(ctx context.Context, name string, config RetryConfig, operation func() error) error {
|
||||
retryer := rw.manager.GetOrCreate(name+".custom", config)
|
||||
return retryer.Execute(ctx, operation)
|
||||
}
|
||||
|
||||
// GetManager 获取重试管理器
|
||||
func (rw *RetryerWrapper) GetManager() *RetryManager {
|
||||
return rw.manager
|
||||
}
|
||||
|
||||
// GetAllStats 获取所有统计信息
|
||||
func (rw *RetryerWrapper) GetAllStats() map[string]RetryStats {
|
||||
return rw.manager.GetStats()
|
||||
}
|
||||
|
||||
// ResetAllStats 重置所有统计信息
|
||||
func (rw *RetryerWrapper) ResetAllStats() {
|
||||
rw.manager.ResetAll()
|
||||
}
|
||||
Reference in New Issue
Block a user