From b92b702f72fff9d4d604d655e16889ac52146b25 Mon Sep 17 00:00:00 2001 From: Shweta Jain Date: Sun, 21 Jan 2018 20:29:16 +0530 Subject: [PATCH] Modified retry logic and resolved callback error --- lib/loggly/common.js | 224 +++++++++++++++++++++++++------------------ 1 file changed, 129 insertions(+), 95 deletions(-) diff --git a/lib/loggly/common.js b/lib/loggly/common.js index fab9c15..c01114d 100644 --- a/lib/loggly/common.js +++ b/lib/loggly/common.js @@ -11,7 +11,8 @@ // var arrSize = 100, arrMsg = [], - timerFunction = null; + timerFunction = null, + sendBulkInMs = 5000; // // Variables for buffer array @@ -25,13 +26,32 @@ var arrBufferedMsg = [], var isValidToken = true; // -// Variables for server retry +// attach event id with each event in both bulk and input mode // -var arrRetryLogs = [], - maxRetryAllowed = 5, - totalRetries = 0, - statusCode, - notFailedOnServerError = true; +var bulkId = 1, + inputId = 1; + +// +// Variables for error retry +// +var numberOfRetries = 5, + eventRetried = 2, + sleepTimeMs, + responseCode; + +// +// Object to hold status codes +// +var httpStatusCode = { + badToken: { + message: 'Forbidden', + code: 403 + }, + success: { + message: 'Success', + code: 200 + } +} var https = require('https'), util = require('util'), @@ -40,35 +60,6 @@ var https = require('https'), var common = exports; -// -// Failure HTTP Response codes based -// off Loggly specification. -// -var failCodes = common.failCodes = { - 400: 'Bad Request', - 401: 'Unauthorized', - 403: 'Forbidden', - 404: 'Not Found', - 409: 'Conflict / Duplicate', - 410: 'Gone', - 500: 'Internal Server Error', - 501: 'Not Implemented', - 503: 'Throttled', - 504: 'Gateway Timeout' -}; - -// -// Success HTTP Response codes based -// off Loggly specification. -// -var successCodes = common.successCodes = { - 200: 'OK', - 201: 'Created', - 202: 'Accepted', - 203: 'Non-authoritative information', - 204: 'Deleted' -}; - // // Core method that actually sends requests to Loggly. // This method is designed to be flexible w.r.t. arguments @@ -165,41 +156,66 @@ common.loggly = function () { if (auth) { requestOptions.headers.authorization = 'Basic ' + new Buffer(auth.username + ':' + auth.password).toString('base64'); } - if (requestBody) { + function popMsgsAndSend() { + if (isBulk) { + var bulk = createBulk(arrMsg); + sendBulkLogs(bulk); + } else { + var input = createInput(requestBody); + sendInputLogs(input); + } + } + function createBulk(msgs) { + var bulkMsg = {}; + bulkMsg.msgs = arrMsg.slice(); + bulkMsg.attemptNumber = 1; + bulkMsg.sleepUntilNextRetry = 2 * 1000; + bulkMsg.id = bulkId++; + + return bulkMsg; + } + function createInput(msgs) { + var inputMsg = {}; + inputMsg.msgs = requestBody; requestOptions.body = requestBody; - arrRetryLogs = arrRetryLogs.concat(requestBody); + inputMsg.attemptNumber = 1; + inputMsg.sleepUntilNextRetry = 2 * 1000; + inputMsg.id = inputId++; + + return inputMsg; } - function sendLogs() { - if (arrRetryLogs.length && !requestBody) requestOptions.body = arrRetryLogs[0]; + function sendInputLogs(input) { try { request(requestOptions, function (err, res, body) { - if (err) return onError(err); - statusCode = res.statusCode.toString(); - if(statusCode === '403') isValidToken = false; - if (statusCode === '500' || statusCode === '503' || statusCode === '504') retryOnServerError(res); - if (statusCode === '200') { - arrRetryLogs.splice(0, 1); - totalRetries = 0; - } - if (Object.keys(failCodes).indexOf(statusCode) !== -1) { - if (statusCode !== '503' && statusCode !== '500' && statusCode !== '504') { - return onError((new Error('Loggly Error (' + statusCode + '): ' + failCodes[statusCode]))); + if (err) { + // In rare cases server is busy + if (err.code === 'ETIMEDOUT' || err.code === 'ECONNRESET' || err.code === 'ESOCKETTIMEDOUT' || err.code === 'ECONNABORTED') { + retryOnError(input, err); + } else { + return onError(err); + } + } else { + responseCode = res.statusCode; + if (responseCode === httpStatusCode.badToken.code) { + isValidToken = false; + return onError((new Error('Loggly Error (' + responseCode + '): ' + httpStatusCode.badToken.message))); + } + if (responseCode === httpStatusCode.success.code && input.attemptNumber >= eventRetried) { + console.log('log #' + input.id + ' sent successfully after ' + input.attemptNumber + ' retries'); + } + if (responseCode === httpStatusCode.success.code) { + success(res, body); + } else { + retryOnError(input, res); + } } - } - success(res, body); }); } catch (ex) { onError(ex); } } - function sendBulkLogs() { - if (arrMsg.length === 0 && arrRetryLogs.length === 0) return; - var retryLogs = []; - if (arrRetryLogs.length && !arrMsg.length) { - retryLogs = arrRetryLogs.slice(0, arrSize); - requestOptions.body = retryLogs.join('\n'); - } + function sendBulkLogs(bulk) { // // Join Array Message with new line ('\n') character // @@ -209,20 +225,28 @@ common.loggly = function () { } try { request(requestOptions, function (err, res, body) { - if (err) return onError(err); - var statusCode = res.statusCode.toString(); - if(statusCode === '403') isValidToken = false; - if (statusCode === '500' || statusCode === '503' || statusCode === '504') retryOnServerError(res); - if (statusCode === '200') { - arrRetryLogs.splice(0, arrSize); - totalRetries = 0; - } - if (Object.keys(failCodes).indexOf(statusCode) !== -1) { - if (statusCode !== '503' && statusCode !== '500' && statusCode !== '504') { - return onError((new Error('Loggly Error (' + statusCode + '): ' + failCodes[statusCode]))); + if (err) { + // In rare cases server is busy + if (err.code === 'ETIMEDOUT' || err.code === 'ECONNRESET' || err.code === 'ESOCKETTIMEDOUT' || err.code === 'ECONNABORTED') { + retryOnError(bulk, err); + } else { + return onError(err); + } + } else { + responseCode = res.statusCode; + if (responseCode === httpStatusCode.badToken.code) { + isValidToken = false; + return onError((new Error('Loggly Error (' + responseCode + '): ' + httpStatusCode.badToken.message))); + } + if (responseCode === httpStatusCode.success.code && bulk.attemptNumber >= eventRetried) { + console.log('log #' + bulk.id + ' sent successfully after ' + bulk.attemptNumber + ' retries'); + } + if (responseCode === httpStatusCode.success.code) { + success(res, body); + } else { + retryOnError(bulk, res); + } } - } - success(res, body); }); } catch (ex) { @@ -232,12 +256,12 @@ common.loggly = function () { if (isBulk && isValidToken) { if (timerFunction === null) { timerFunction = setInterval(function () { - sendBulkLogs(); + if (arrMsg.length) popMsgsAndSend(); if (timerFunction && !arrMsg.length) { clearInterval(timerFunction) timerFunction = null; } - },5000); + }, sendBulkInMs); } if (Array.isArray(requestBody)) { @@ -247,32 +271,42 @@ common.loggly = function () { } if (arrMsg.length === arrSize) { - sendBulkLogs(); + popMsgsAndSend(); } } else if(isValidToken) { - sendLogs(); + if (requestBody) { + popMsgsAndSend(); + } } // - //function to retry sending logs maximum 5 times if server error occurs + //function to retry sending logs maximum 5 times if any error occurs // - function retryOnServerError(err) { - if (!arrRetryLogs.length) return; - else { - if (notFailedOnServerError && totalRetries >= maxRetryAllowed) { - console.log('Failed after ' + totalRetries + ' retries on error - ' + statusCode, '"'+err.statusMessage+'"'); - notFailedOnServerError = false; - arrRetryLogs.length = 0; - totalRetries = 0; + function retryOnError(mode, response) { + function tryAgainIn(sleepTimeMs) { + console.log('log #' + mode.id + ' - Trying again in ' + sleepTimeMs + '[ms], attempt no. ' + mode.attemptNumber); + setTimeout(function () { + isBulk ? sendBulkLogs(mode) : sendInputLogs(mode); + }, sleepTimeMs); + } + if (mode.attemptNumber >= numberOfRetries) { + if (response.code) { + console.error('Failed log #' + mode.id + ' after ' + mode.attemptNumber + ' retries on error = ' + response, response); + } else { + console.error('Failed log #' + mode.id + ' after ' + mode.attemptNumber + ' retries on error = ' + response.statusCode + ' ' + response.statusMessage); } - while (isValidToken && totalRetries < maxRetryAllowed) { - console.log('Failed on error code ' + statusCode); - console.log('Retried ' + (totalRetries + 1) + ' time'); - totalRetries++; - isBulk ? sendBulkLogs() : sendLogs(); + } else { + if (response.code) { + console.log('log #' + mode.id + ' - failed on error: ' + response); + } else { + console.log('log #' + mode.id + ' - failed on error: ' + response.statusCode + ' ' + response.statusMessage); + } + sleepTimeMs = mode.sleepUntilNextRetry; + mode.sleepUntilNextRetry = mode.sleepUntilNextRetry * 2; + mode.attemptNumber++; + tryAgainIn(sleepTimeMs) } - } } // @@ -299,8 +333,8 @@ common.loggly = function () { requestOptionsForBufferedLogs.body = isBulk ? arrayMessage.join('\n') : arrayMessage[0]; request(requestOptionsForBufferedLogs, function (err, res, body) { if(err) return; - statusCode = res.statusCode.toString(); - if(statusCode === "200") { + statusCode = res.statusCode; + if(statusCode === httpStatusCode.success.code) { arrBufferedMsg.splice(0, logsInBunch); sendBufferdLogstoLoggly(); } @@ -316,7 +350,7 @@ common.loggly = function () { var numberOfLogsToBeRemoved = (arrBufferedMsg.length + logs.length) - bufferOptions.size; if (numberOfLogsToBeRemoved > 0) arrBufferedMsg = arrBufferedMsg.splice(numberOfLogsToBeRemoved); arrBufferedMsg = arrBufferedMsg.concat(logs); - } + } }; // // ### function serialize (obj, key)