From 635ae4359724e61fcedb2ebab87c2f4cbcfabaf6 Mon Sep 17 00:00:00 2001 From: Shweta Jain Date: Tue, 24 Oct 2017 15:15:29 +0530 Subject: [PATCH 1/2] Added code to retry on server error --- lib/loggly/common.js | 82 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 67 insertions(+), 15 deletions(-) diff --git a/lib/loggly/common.js b/lib/loggly/common.js index 6db51a0..42bb657 100644 --- a/lib/loggly/common.js +++ b/lib/loggly/common.js @@ -14,7 +14,7 @@ var arrSize = 100, timerFunction = null; // -// Variables for buffer array +// Variables for buffer array // var arrBufferedMsg = [], timerFunctionForBufferedLogs = null; @@ -24,6 +24,15 @@ var arrBufferedMsg = [], // var isValidToken = true; +// +// Variables for server retry +// +var arrRetryLogs = [], +maxRetryAllowed = 5, +totalRetries = 0, +statusCode, +notfailedOnServerError = true; + var https = require('https'), util = require('util'), request = require('request'), @@ -44,7 +53,9 @@ var failCodes = common.failCodes = { 410: 'Gone', 500: 'Internal Server Error', 501: 'Not Implemented', - 503: 'Throttled' + 503: 'Throttled', + 503: 'Throttled', + 504: 'Gateway Timeout' }; // @@ -137,6 +148,7 @@ common.loggly = function () { arrayLogs.push(requestOptions.body); } storeLogs(arrayLogs); + //storeLogsforServerRetry(arrayLogs); if (!responded) { responded = true; @@ -157,18 +169,28 @@ common.loggly = function () { } if (requestBody) { requestOptions.body = requestBody; + arrRetryLogs = arrRetryLogs.concat(requestBody); + } + if (arrRetryLogs.length && !requestBody) { + requestOptions.body = arrRetryLogs[0]; } function sendLogs() { try { request(requestOptions, function (err, res, body) { - if (err) { - return onError(err); - } - var statusCode = res.statusCode.toString(); + if (err) return onError(err); + + statusCode = res.statusCode.toString(); if(statusCode === '403') isValidToken = false; + if (statusCode === '200') { + arrRetryLogs.splice(0, 1); + totalRetries = 0; + } + if (statusCode === '500' || statusCode === '503' || statusCode === '504') retryOnServerError(res); if (Object.keys(failCodes).indexOf(statusCode) !== -1) { - return onError((new Error('Loggly Error (' + statusCode + '): ' + failCodes[statusCode]))); - } + if (statusCode !== '503' && statusCode !== '500' && statusCode !== '504') { + return onError((new Error('Loggly Error (' + statusCode + '): ' + failCodes[statusCode]))); + } + } success(res, body); }); } @@ -177,23 +199,32 @@ common.loggly = function () { } } function sendBulkLogs() { - if (arrMsg.length === 0) { - return; + // if (arrMsg.length === 0) { + // return; + // } + if (arrRetryLogs.length && !arrMsg.length) { + requestOptions.body = arrRetryLogs.join('\n'); } // // Join Array Message with new line ('\n') character // - requestOptions.body = arrMsg.join('\n'); + if (arrMsg.length) requestOptions.body = arrMsg.join('\n'); arrMsg.length = 0; try { request(requestOptions, function (err, res, body) { - if (err) { - return onError(err); - } + if (err) return onError(err); + var statusCode = res.statusCode.toString(); if(statusCode === '403') isValidToken = false; + if (statusCode === '200') { + arrRetryLogs.splice(0, arrSize); + totalRetries = 0; + } + if (statusCode === '500' || statusCode === '503' || statusCode === '504') retryOnServerError(res); if (Object.keys(failCodes).indexOf(statusCode) !== -1) { - return onError((new Error('Loggly Error (' + statusCode + '): ' + failCodes[statusCode]))); + if (statusCode !== '503' && statusCode !== '500' && statusCode !== '504') { + return onError((new Error('Loggly Error (' + statusCode + '): ' + failCodes[statusCode]))); + } } success(res, body); }); @@ -227,6 +258,20 @@ common.loggly = function () { sendLogs(); } + function retryOnServerError(err) { + if (!arrRetryLogs) return; + if (notfailedOnServerError && totalRetries >= maxRetryAllowed) { + console.log('Failed after ' + totalRetries + ' retries on error - ' + statusCode, err.statusMessage); + notfailedOnServerError = false; + } + while (isValidToken && totalRetries < maxRetryAllowed) { + console.log('Failed on error code ' + statusCode); + console.log('Retried ' + (totalRetries + 1) + ' time'); + totalRetries++; + isBulk ? sendBulkLogs() : sendLogs(); + } + } + // // retries to send buffered logs to loggly in every 30 seconds // @@ -269,6 +314,13 @@ common.loggly = function () { if (numberOfLogsToBeRemoved > 0) arrBufferedMsg = arrBufferedMsg.splice(numberOfLogsToBeRemoved); arrBufferedMsg = arrBufferedMsg.concat(logs); } + + //function storeLogsforServerRetry(logs) { + // if (!logs.length || !bufferOptions) return; + // arrRetryLogs = arrRetryLogs.concat(logs); + // isBulk ? sendBulkLogs(arrRetryLogs) : sendLogs(arrRetryLogs); + // //sendLogs(arrInputLogs); + // } }; // // ### function serialize (obj, key) From 6ded6c80f2e20b308607493d129129383abe005d Mon Sep 17 00:00:00 2001 From: Shweta Jain Date: Tue, 31 Oct 2017 18:05:35 +0530 Subject: [PATCH 2/2] modified server retry code --- lib/loggly/common.js | 70 +++++++++++++++++++++----------------------- 1 file changed, 33 insertions(+), 37 deletions(-) diff --git a/lib/loggly/common.js b/lib/loggly/common.js index 42bb657..fab9c15 100644 --- a/lib/loggly/common.js +++ b/lib/loggly/common.js @@ -28,10 +28,10 @@ var isValidToken = true; // Variables for server retry // var arrRetryLogs = [], -maxRetryAllowed = 5, -totalRetries = 0, -statusCode, -notfailedOnServerError = true; + maxRetryAllowed = 5, + totalRetries = 0, + statusCode, + notFailedOnServerError = true; var https = require('https'), util = require('util'), @@ -54,7 +54,6 @@ var failCodes = common.failCodes = { 500: 'Internal Server Error', 501: 'Not Implemented', 503: 'Throttled', - 503: 'Throttled', 504: 'Gateway Timeout' }; @@ -148,7 +147,6 @@ common.loggly = function () { arrayLogs.push(requestOptions.body); } storeLogs(arrayLogs); - //storeLogsforServerRetry(arrayLogs); if (!responded) { responded = true; @@ -171,21 +169,18 @@ common.loggly = function () { requestOptions.body = requestBody; arrRetryLogs = arrRetryLogs.concat(requestBody); } - if (arrRetryLogs.length && !requestBody) { - requestOptions.body = arrRetryLogs[0]; - } function sendLogs() { + if (arrRetryLogs.length && !requestBody) requestOptions.body = arrRetryLogs[0]; try { request(requestOptions, function (err, res, body) { if (err) return onError(err); - statusCode = res.statusCode.toString(); if(statusCode === '403') isValidToken = false; + if (statusCode === '500' || statusCode === '503' || statusCode === '504') retryOnServerError(res); if (statusCode === '200') { arrRetryLogs.splice(0, 1); totalRetries = 0; } - if (statusCode === '500' || statusCode === '503' || statusCode === '504') retryOnServerError(res); if (Object.keys(failCodes).indexOf(statusCode) !== -1) { if (statusCode !== '503' && statusCode !== '500' && statusCode !== '504') { return onError((new Error('Loggly Error (' + statusCode + '): ' + failCodes[statusCode]))); @@ -199,28 +194,29 @@ common.loggly = function () { } } function sendBulkLogs() { - // if (arrMsg.length === 0) { - // return; - // } + if (arrMsg.length === 0 && arrRetryLogs.length === 0) return; + var retryLogs = []; if (arrRetryLogs.length && !arrMsg.length) { - requestOptions.body = arrRetryLogs.join('\n'); + retryLogs = arrRetryLogs.slice(0, arrSize); + requestOptions.body = retryLogs.join('\n'); } // // Join Array Message with new line ('\n') character // - if (arrMsg.length) requestOptions.body = arrMsg.join('\n'); - arrMsg.length = 0; + if (arrMsg.length) { + requestOptions.body = arrMsg.join('\n'); + arrMsg.length = 0; + } try { request(requestOptions, function (err, res, body) { if (err) return onError(err); - var statusCode = res.statusCode.toString(); if(statusCode === '403') isValidToken = false; + if (statusCode === '500' || statusCode === '503' || statusCode === '504') retryOnServerError(res); if (statusCode === '200') { arrRetryLogs.splice(0, arrSize); totalRetries = 0; } - if (statusCode === '500' || statusCode === '503' || statusCode === '504') retryOnServerError(res); if (Object.keys(failCodes).indexOf(statusCode) !== -1) { if (statusCode !== '503' && statusCode !== '500' && statusCode !== '504') { return onError((new Error('Loggly Error (' + statusCode + '): ' + failCodes[statusCode]))); @@ -258,17 +254,24 @@ common.loggly = function () { sendLogs(); } + // + //function to retry sending logs maximum 5 times if server error occurs + // function retryOnServerError(err) { - if (!arrRetryLogs) return; - if (notfailedOnServerError && totalRetries >= maxRetryAllowed) { - console.log('Failed after ' + totalRetries + ' retries on error - ' + statusCode, err.statusMessage); - notfailedOnServerError = false; - } - while (isValidToken && totalRetries < maxRetryAllowed) { - console.log('Failed on error code ' + statusCode); - console.log('Retried ' + (totalRetries + 1) + ' time'); - totalRetries++; - isBulk ? sendBulkLogs() : sendLogs(); + if (!arrRetryLogs.length) return; + else { + if (notFailedOnServerError && totalRetries >= maxRetryAllowed) { + console.log('Failed after ' + totalRetries + ' retries on error - ' + statusCode, '"'+err.statusMessage+'"'); + notFailedOnServerError = false; + arrRetryLogs.length = 0; + totalRetries = 0; + } + while (isValidToken && totalRetries < maxRetryAllowed) { + console.log('Failed on error code ' + statusCode); + console.log('Retried ' + (totalRetries + 1) + ' time'); + totalRetries++; + isBulk ? sendBulkLogs() : sendLogs(); + } } } @@ -296,7 +299,7 @@ common.loggly = function () { requestOptionsForBufferedLogs.body = isBulk ? arrayMessage.join('\n') : arrayMessage[0]; request(requestOptionsForBufferedLogs, function (err, res, body) { if(err) return; - var statusCode = res.statusCode.toString(); + statusCode = res.statusCode.toString(); if(statusCode === "200") { arrBufferedMsg.splice(0, logsInBunch); sendBufferdLogstoLoggly(); @@ -314,13 +317,6 @@ common.loggly = function () { if (numberOfLogsToBeRemoved > 0) arrBufferedMsg = arrBufferedMsg.splice(numberOfLogsToBeRemoved); arrBufferedMsg = arrBufferedMsg.concat(logs); } - - //function storeLogsforServerRetry(logs) { - // if (!logs.length || !bufferOptions) return; - // arrRetryLogs = arrRetryLogs.concat(logs); - // isBulk ? sendBulkLogs(arrRetryLogs) : sendLogs(arrRetryLogs); - // //sendLogs(arrInputLogs); - // } }; // // ### function serialize (obj, key)