backoff.go 5.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187
  1. /*
  2. Copyright 2015 The Kubernetes Authors.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. */
  13. package flowcontrol
  14. import (
  15. "math/rand"
  16. "sync"
  17. "time"
  18. "k8s.io/utils/clock"
  19. )
  20. type backoffEntry struct {
  21. backoff time.Duration
  22. lastUpdate time.Time
  23. }
  24. type Backoff struct {
  25. sync.RWMutex
  26. Clock clock.Clock
  27. // HasExpiredFunc controls the logic that determines whether the backoff
  28. // counter should be reset, and when to GC old backoff entries. If nil, the
  29. // default hasExpired function will restart the backoff factor to the
  30. // beginning after observing time has passed at least equal to 2*maxDuration
  31. HasExpiredFunc func(eventTime time.Time, lastUpdate time.Time, maxDuration time.Duration) bool
  32. defaultDuration time.Duration
  33. maxDuration time.Duration
  34. perItemBackoff map[string]*backoffEntry
  35. rand *rand.Rand
  36. // maxJitterFactor adds jitter to the exponentially backed off delay.
  37. // if maxJitterFactor is zero, no jitter is added to the delay in
  38. // order to maintain current behavior.
  39. maxJitterFactor float64
  40. }
  41. func NewFakeBackOff(initial, max time.Duration, tc clock.Clock) *Backoff {
  42. return newBackoff(tc, initial, max, 0.0)
  43. }
  44. func NewBackOff(initial, max time.Duration) *Backoff {
  45. return NewBackOffWithJitter(initial, max, 0.0)
  46. }
  47. func NewFakeBackOffWithJitter(initial, max time.Duration, tc clock.Clock, maxJitterFactor float64) *Backoff {
  48. return newBackoff(tc, initial, max, maxJitterFactor)
  49. }
  50. func NewBackOffWithJitter(initial, max time.Duration, maxJitterFactor float64) *Backoff {
  51. clock := clock.RealClock{}
  52. return newBackoff(clock, initial, max, maxJitterFactor)
  53. }
  54. func newBackoff(clock clock.Clock, initial, max time.Duration, maxJitterFactor float64) *Backoff {
  55. var random *rand.Rand
  56. if maxJitterFactor > 0 {
  57. random = rand.New(rand.NewSource(clock.Now().UnixNano()))
  58. }
  59. return &Backoff{
  60. perItemBackoff: map[string]*backoffEntry{},
  61. Clock: clock,
  62. defaultDuration: initial,
  63. maxDuration: max,
  64. maxJitterFactor: maxJitterFactor,
  65. rand: random,
  66. }
  67. }
  68. // Get the current backoff Duration
  69. func (p *Backoff) Get(id string) time.Duration {
  70. p.RLock()
  71. defer p.RUnlock()
  72. var delay time.Duration
  73. entry, ok := p.perItemBackoff[id]
  74. if ok {
  75. delay = entry.backoff
  76. }
  77. return delay
  78. }
  79. // move backoff to the next mark, capping at maxDuration
  80. func (p *Backoff) Next(id string, eventTime time.Time) {
  81. p.Lock()
  82. defer p.Unlock()
  83. entry, ok := p.perItemBackoff[id]
  84. if !ok || p.hasExpired(eventTime, entry.lastUpdate, p.maxDuration) {
  85. entry = p.initEntryUnsafe(id)
  86. entry.backoff += p.jitter(entry.backoff)
  87. } else {
  88. delay := entry.backoff * 2 // exponential
  89. delay += p.jitter(entry.backoff) // add some jitter to the delay
  90. entry.backoff = min(delay, p.maxDuration)
  91. }
  92. entry.lastUpdate = p.Clock.Now()
  93. }
  94. // Reset forces clearing of all backoff data for a given key.
  95. func (p *Backoff) Reset(id string) {
  96. p.Lock()
  97. defer p.Unlock()
  98. delete(p.perItemBackoff, id)
  99. }
  100. // Returns True if the elapsed time since eventTime is smaller than the current backoff window
  101. func (p *Backoff) IsInBackOffSince(id string, eventTime time.Time) bool {
  102. p.RLock()
  103. defer p.RUnlock()
  104. entry, ok := p.perItemBackoff[id]
  105. if !ok {
  106. return false
  107. }
  108. if p.hasExpired(eventTime, entry.lastUpdate, p.maxDuration) {
  109. return false
  110. }
  111. return p.Clock.Since(eventTime) < entry.backoff
  112. }
  113. // Returns True if time since lastupdate is less than the current backoff window.
  114. func (p *Backoff) IsInBackOffSinceUpdate(id string, eventTime time.Time) bool {
  115. p.RLock()
  116. defer p.RUnlock()
  117. entry, ok := p.perItemBackoff[id]
  118. if !ok {
  119. return false
  120. }
  121. if p.hasExpired(eventTime, entry.lastUpdate, p.maxDuration) {
  122. return false
  123. }
  124. return eventTime.Sub(entry.lastUpdate) < entry.backoff
  125. }
  126. // Garbage collect records that have aged past their expiration, which defaults
  127. // to 2*maxDuration (see hasExpired godoc). Backoff users are expected to invoke
  128. // this periodically.
  129. func (p *Backoff) GC() {
  130. p.Lock()
  131. defer p.Unlock()
  132. now := p.Clock.Now()
  133. for id, entry := range p.perItemBackoff {
  134. if p.hasExpired(now, entry.lastUpdate, p.maxDuration) {
  135. delete(p.perItemBackoff, id)
  136. }
  137. }
  138. }
  139. func (p *Backoff) DeleteEntry(id string) {
  140. p.Lock()
  141. defer p.Unlock()
  142. delete(p.perItemBackoff, id)
  143. }
  144. // Take a lock on *Backoff, before calling initEntryUnsafe
  145. func (p *Backoff) initEntryUnsafe(id string) *backoffEntry {
  146. entry := &backoffEntry{backoff: p.defaultDuration}
  147. p.perItemBackoff[id] = entry
  148. return entry
  149. }
  150. func (p *Backoff) jitter(delay time.Duration) time.Duration {
  151. if p.rand == nil {
  152. return 0
  153. }
  154. return time.Duration(p.rand.Float64() * p.maxJitterFactor * float64(delay))
  155. }
  156. // Unless an alternate function is provided, after 2*maxDuration we restart the backoff factor to the beginning
  157. func (p *Backoff) hasExpired(eventTime time.Time, lastUpdate time.Time, maxDuration time.Duration) bool {
  158. if p.HasExpiredFunc != nil {
  159. return p.HasExpiredFunc(eventTime, lastUpdate, maxDuration)
  160. }
  161. return eventTime.Sub(lastUpdate) > maxDuration*2 // consider stable if it's ok for twice the maxDuration
  162. }