diff --git a/lib/loggly/common.js b/lib/loggly/common.js index 6db51a0..fab9c15 100644 --- a/lib/loggly/common.js +++ b/lib/loggly/common.js @@ -14,7 +14,7 @@ var arrSize = 100, timerFunction = null; // -// Variables for buffer array +// Variables for buffer array // var arrBufferedMsg = [], timerFunctionForBufferedLogs = null; @@ -24,6 +24,15 @@ var arrBufferedMsg = [], // var isValidToken = true; +// +// Variables for server retry +// +var arrRetryLogs = [], + maxRetryAllowed = 5, + totalRetries = 0, + statusCode, + notFailedOnServerError = true; + var https = require('https'), util = require('util'), request = require('request'), @@ -44,7 +53,8 @@ var failCodes = common.failCodes = { 410: 'Gone', 500: 'Internal Server Error', 501: 'Not Implemented', - 503: 'Throttled' + 503: 'Throttled', + 504: 'Gateway Timeout' }; // @@ -157,18 +167,25 @@ common.loggly = function () { } if (requestBody) { requestOptions.body = requestBody; + arrRetryLogs = arrRetryLogs.concat(requestBody); } function sendLogs() { + if (arrRetryLogs.length && !requestBody) requestOptions.body = arrRetryLogs[0]; try { request(requestOptions, function (err, res, body) { - if (err) { - return onError(err); - } - var statusCode = res.statusCode.toString(); + if (err) return onError(err); + statusCode = res.statusCode.toString(); if(statusCode === '403') isValidToken = false; + if (statusCode === '500' || statusCode === '503' || statusCode === '504') retryOnServerError(res); + if (statusCode === '200') { + arrRetryLogs.splice(0, 1); + totalRetries = 0; + } if (Object.keys(failCodes).indexOf(statusCode) !== -1) { - return onError((new Error('Loggly Error (' + statusCode + '): ' + failCodes[statusCode]))); - } + if (statusCode !== '503' && statusCode !== '500' && statusCode !== '504') { + return onError((new Error('Loggly Error (' + statusCode + '): ' + failCodes[statusCode]))); + } + } success(res, body); }); } @@ -177,23 +194,33 @@ common.loggly = function () { } } function sendBulkLogs() { - if (arrMsg.length === 0) { - return; + if (arrMsg.length === 0 && arrRetryLogs.length === 0) return; + var retryLogs = []; + if (arrRetryLogs.length && !arrMsg.length) { + retryLogs = arrRetryLogs.slice(0, arrSize); + requestOptions.body = retryLogs.join('\n'); } // // Join Array Message with new line ('\n') character // - requestOptions.body = arrMsg.join('\n'); - arrMsg.length = 0; + if (arrMsg.length) { + requestOptions.body = arrMsg.join('\n'); + arrMsg.length = 0; + } try { request(requestOptions, function (err, res, body) { - if (err) { - return onError(err); - } + if (err) return onError(err); var statusCode = res.statusCode.toString(); if(statusCode === '403') isValidToken = false; + if (statusCode === '500' || statusCode === '503' || statusCode === '504') retryOnServerError(res); + if (statusCode === '200') { + arrRetryLogs.splice(0, arrSize); + totalRetries = 0; + } if (Object.keys(failCodes).indexOf(statusCode) !== -1) { - return onError((new Error('Loggly Error (' + statusCode + '): ' + failCodes[statusCode]))); + if (statusCode !== '503' && statusCode !== '500' && statusCode !== '504') { + return onError((new Error('Loggly Error (' + statusCode + '): ' + failCodes[statusCode]))); + } } success(res, body); }); @@ -227,6 +254,27 @@ common.loggly = function () { sendLogs(); } + // + //function to retry sending logs maximum 5 times if server error occurs + // + function retryOnServerError(err) { + if (!arrRetryLogs.length) return; + else { + if (notFailedOnServerError && totalRetries >= maxRetryAllowed) { + console.log('Failed after ' + totalRetries + ' retries on error - ' + statusCode, '"'+err.statusMessage+'"'); + notFailedOnServerError = false; + arrRetryLogs.length = 0; + totalRetries = 0; + } + while (isValidToken && totalRetries < maxRetryAllowed) { + console.log('Failed on error code ' + statusCode); + console.log('Retried ' + (totalRetries + 1) + ' time'); + totalRetries++; + isBulk ? sendBulkLogs() : sendLogs(); + } + } + } + // // retries to send buffered logs to loggly in every 30 seconds // @@ -251,7 +299,7 @@ common.loggly = function () { requestOptionsForBufferedLogs.body = isBulk ? arrayMessage.join('\n') : arrayMessage[0]; request(requestOptionsForBufferedLogs, function (err, res, body) { if(err) return; - var statusCode = res.statusCode.toString(); + statusCode = res.statusCode.toString(); if(statusCode === "200") { arrBufferedMsg.splice(0, logsInBunch); sendBufferdLogstoLoggly();