From 3197a27d24248a1dd3c951172c209f4710204fb6 Mon Sep 17 00:00:00 2001 From: Kevin Locke Date: Fri, 3 Feb 2017 23:21:05 -0700 Subject: [PATCH] Add failAfterTime to limit maximum time elapsed This commit adds the failAfterTime method to Backoff (and FunctionCall) to allow limiting the total elapsed time. The primary intended use case is when the caller has a deadline which can not be exceeded during which to retry an operation with backoff and the number of backoffs can not be calculated in advance (e.g. because the operation takes a variable amount of time). Signed-off-by: Kevin Locke --- README.md | 10 +++++ examples/fail_time.js | 34 +++++++++++++++ lib/backoff.js | 25 ++++++++++- lib/function_call.js | 11 +++++ tests/backoff.js | 96 ++++++++++++++++++++++++++++++++++++++++++ tests/function_call.js | 28 ++++++++++++ 6 files changed, 202 insertions(+), 2 deletions(-) create mode 100755 examples/fail_time.js diff --git a/README.md b/README.md index 525f7ba..79d6f16 100644 --- a/README.md +++ b/README.md @@ -168,6 +168,16 @@ Sets a limit on the maximum number of backoffs that can be performed before a fail event gets emitted and the backoff instance is reset. By default, there is no limit on the number of backoffs that can be performed. +#### backoff.failAfterTime(maxTotalTime) + +- maxTotalTime: maximum time (in milliseconds) before the fail event gets +emitted upon backoff, must be greater than 0 + +Sets a limit on the maximum amount of time from the start of the first backoff +before attempting to backoff will emit a fail event and reset the backoff +instance. The last backoff before the limit is reached will be truncated to +avoid exceeding the limit. By default, there is no time limit. + #### backoff.backoff([err]) Starts a backoff operation. If provided, the error parameter will be emitted diff --git a/examples/fail_time.js b/examples/fail_time.js new file mode 100755 index 0000000..64dc73c --- /dev/null +++ b/examples/fail_time.js @@ -0,0 +1,34 @@ +#!/usr/bin/env node + +var backoff = require('../index'); + +var testBackoff = backoff.exponential({ + initialDelay: 10, + maxDelay: 1000 +}); + +testBackoff.failAfterTime(600); + +var start; +testBackoff.on('backoff', function(number, delay) { + console.log('Backoff start: ' + number + ' ' + delay + 'ms' + + ' (' + (Date.now() - start) + 'ms elapsed)'); +}); + +var callDelay = 50; +testBackoff.on('ready', function(number, delay) { + console.log('Backoff done: ' + number + ' ' + delay + 'ms' + + ' (' + (Date.now() - start) + 'ms elapsed)'); + setTimeout(function() { + console.log('Simulated call delay: ' + callDelay + 'ms' + + ' (' + (Date.now() - start) + 'ms elapsed)'); + testBackoff.backoff(); // Launch a new backoff. + }, callDelay); +}); + +testBackoff.on('fail', function() { + console.log('Backoff failure.'); +}); + +start = Date.now(); +testBackoff.backoff(); diff --git a/lib/backoff.js b/lib/backoff.js index 202a280..bda5680 100644 --- a/lib/backoff.js +++ b/lib/backoff.js @@ -12,7 +12,9 @@ function Backoff(backoffStrategy) { this.backoffStrategy_ = backoffStrategy; this.maxNumberOfRetry_ = -1; + this.maxTotalTime_ = Infinity; this.backoffNumber_ = 0; + this.backoffStart_ = -1; this.backoffDelay_ = 0; this.timeoutID_ = -1; @@ -32,16 +34,34 @@ Backoff.prototype.failAfter = function(maxNumberOfRetry) { this.maxNumberOfRetry_ = maxNumberOfRetry; }; +// Sets a time limit, in milliseconds, greater than 0, on the maximum time from +// the first backoff. A 'fail' event will be emitted when a backoff is +// attempted after the limit is reached. The limit may shorten the last +// backoff before the time limit is reached to avoid exceeding the limit. +Backoff.prototype.failAfterTime = function(maxTotalTime) { + precond.checkArgument(maxTotalTime > 0, + 'Expected a maximum total time greater than 0 but got %s.', + maxTotalTime); + + this.maxTotalTime_ = maxTotalTime; +}; + // Starts a backoff operation. Accepts an optional parameter to let the // listeners know why the backoff operation was started. Backoff.prototype.backoff = function(err) { precond.checkState(this.timeoutID_ === -1, 'Backoff in progress.'); - if (this.backoffNumber_ === this.maxNumberOfRetry_) { + var now = Date.now(); + if (this.backoffStart_ === -1) { + this.backoffStart_ = now; + } + + var timeLeft = this.maxTotalTime_ - (now - this.backoffStart_); + if (this.backoffNumber_ === this.maxNumberOfRetry_ || timeLeft <= 0) { this.emit('fail', err); this.reset(); } else { - this.backoffDelay_ = this.backoffStrategy_.next(); + this.backoffDelay_ = Math.min(this.backoffStrategy_.next(), timeLeft); this.timeoutID_ = setTimeout(this.handlers.backoff, this.backoffDelay_); this.emit('backoff', this.backoffNumber_, this.backoffDelay_, err); } @@ -57,6 +77,7 @@ Backoff.prototype.onBackoff_ = function() { // Stops any backoff operation and resets the backoff delay to its inital value. Backoff.prototype.reset = function() { this.backoffNumber_ = 0; + this.backoffStart_ = -1; this.backoffStrategy_.reset(); clearTimeout(this.timeoutID_); this.timeoutID_ = -1; diff --git a/lib/function_call.js b/lib/function_call.js index 37319d7..51cef54 100644 --- a/lib/function_call.js +++ b/lib/function_call.js @@ -25,6 +25,7 @@ function FunctionCall(fn, args, callback) { this.backoff_ = null; this.strategy_ = null; this.failAfter_ = -1; + this.failAfterTime_ = -1; this.retryPredicate_ = FunctionCall.DEFAULT_RETRY_PREDICATE_; this.state_ = FunctionCall.State_.PENDING; @@ -105,6 +106,13 @@ FunctionCall.prototype.failAfter = function(maxNumberOfRetry) { return this; // Return this for chaining. }; +// Sets the backoff time limit. +FunctionCall.prototype.failAfterTime = function(maxTotalTime) { + precond.checkState(this.isPending(), 'FunctionCall in progress.'); + this.failAfterTime_ = maxTotalTime; + return this; // Return this for chaining. +}; + // Aborts the call. FunctionCall.prototype.abort = function() { if (this.isCompleted() || this.isAborted()) { @@ -140,6 +148,9 @@ FunctionCall.prototype.start = function(backoffFactory) { if (this.failAfter_ > 0) { this.backoff_.failAfter(this.failAfter_); } + if (this.failAfterTime_ > 0) { + this.backoff_.failAfterTime(this.failAfterTime_); + } this.state_ = FunctionCall.State_.RUNNING; this.doCall_(false /* isRetry */); diff --git a/tests/backoff.js b/tests/backoff.js index 84bbd16..550fac8 100644 --- a/tests/backoff.js +++ b/tests/backoff.js @@ -91,6 +91,80 @@ exports["Backoff"] = { test.done(); }, + "the fail event should be emitted when time limit is reached": function(test) { + var err = new Error('Fail'); + + this.backoffStrategy.next.returns(10); + this.backoff.on('fail', this.spy); + + this.backoff.failAfterTime(20); + + // Consume first 2 backoffs so limit is reached. + for (var i = 0; i < 2; i++) { + this.backoff.backoff(); + this.clock.tick(10); + } + + // Failure should occur on the third call, and not before. + test.ok(!this.spy.calledOnce, 'Fail event shouldn\'t have been emitted.'); + this.backoff.backoff(err); + test.ok(this.spy.calledOnce, 'Fail event should have been emitted.'); + test.equal(this.spy.getCall(0).args[0], err, 'Error should be passed'); + + test.done(); + }, + + "time limit should include all time": function(test) { + var err = new Error('Fail'); + + this.backoffStrategy.next.returns(10); + this.backoff.on('fail', this.spy); + + this.backoff.failAfterTime(15); + + // Time is started from first backoff. + this.backoff.backoff(); + + this.clock.tick(15); + + // Failure should occur when backoff is called, and not before. + test.ok(!this.spy.calledOnce, 'Fail event shouldn\'t have been emitted.'); + this.backoff.backoff(err); + test.ok(this.spy.calledOnce, 'Fail event should have been emitted.'); + test.equal(this.spy.getCall(0).args[0], err, 'Error should be passed'); + + test.done(); + }, + + "last backoff time should be reduced by time limit": function(test) { + var err = new Error('Fail'); + + this.backoffStrategy.next.returns(10); + + var failSpy = new sinon.spy(); + this.backoff.on('backoff', this.spy); + this.backoff.on('fail', failSpy); + + this.backoff.failAfterTime(25); + + // Consume first 2 backoffs. + for (var i = 0; i < 2; i++) { + this.backoff.backoff(); + this.clock.tick(10); + } + + test.equals(this.spy.callCount, 2, 'Backoff occurs normally before time limit.'); + this.backoff.backoff(); + this.clock.tick(5); + test.equals(this.spy.callCount, 3, 'Last backoff is truncated by the time limit.'); + test.equals(failSpy.callCount, 0, 'Fail does not occur before time limit.'); + this.backoff.backoff(err); + test.ok(failSpy.calledOnce, 'Fail event should have been emitted.'); + test.equal(failSpy.getCall(0).args[0], err, 'Error should be passed'); + + test.done(); + }, + "calling backoff while a backoff is in progress should throw an error": function(test) { this.backoffStrategy.next.returns(10); var backoff = this.backoff; @@ -112,6 +186,14 @@ exports["Backoff"] = { test.done(); }, + "time limit should be greater than 0": function(test) { + var backoff = this.backoff; + test.throws(function() { + backoff.failAfterTime(0); + }, /greater than 0 but got 0/); + test.done(); + }, + "reset should cancel any backoff in progress": function(test) { this.backoffStrategy.next.returns(10); this.backoff.on('ready', this.spy); @@ -146,6 +228,20 @@ exports["Backoff"] = { test.done(); }, + "backoff should be reset after time fail": function(test) { + this.backoffStrategy.next.returns(10); + + this.backoff.failAfterTime(1); + + this.backoff.backoff(); + this.clock.tick(10); + this.backoff.backoff(); + + test.ok(this.backoffStrategy.reset.calledOnce, + 'Backoff should have been resetted after failure.'); + test.done(); + }, + "the backoff number should increase from 0 to N - 1": function(test) { this.backoffStrategy.next.returns(10); this.backoff.on('backoff', this.spy); diff --git a/tests/function_call.js b/tests/function_call.js index d54f4b0..2b26e62 100644 --- a/tests/function_call.js +++ b/tests/function_call.js @@ -16,6 +16,7 @@ function MockBackoff() { this.reset = sinon.spy(); this.backoff = sinon.spy(); this.failAfter = sinon.spy(); + this.failAfterTime = sinon.spy(); } util.inherits(MockBackoff, events.EventEmitter); @@ -158,6 +159,33 @@ exports["FunctionCall"] = { test.done(); }, + "failAfterTime should not be set by default": function(test) { + var call = new FunctionCall(this.wrappedFn, [], this.callback); + call.start(this.backoffFactory); + test.equal(0, this.backoff.failAfterTime.callCount); + test.done(); + }, + + "failAfterTime should be used as the maximum time": function(test) { + var failAfterTimeValue = 99; + var call = new FunctionCall(this.wrappedFn, [], this.callback); + call.failAfterTime(failAfterTimeValue); + call.start(this.backoffFactory); + test.ok(this.backoff.failAfterTime.calledWith(failAfterTimeValue), + 'User defined maximum time shoud be ' + + 'used to configure the backoff instance.'); + test.done(); + }, + + "failAfterTime should throw if the call is in progress": function(test) { + var call = new FunctionCall(this.wrappedFn, [], this.callback); + call.start(this.backoffFactory); + test.throws(function() { + call.failAfterTime(1234); + }, /in progress/); + test.done(); + }, + "start shouldn't allow overlapping invocation": function(test) { var call = new FunctionCall(this.wrappedFn, [], this.callback); var backoffFactory = this.backoffFactory;