From 3a59a7396ebf96fc7f16075f4e3821e03d42800f Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Mon, 27 Oct 2014 22:28:19 +0530 Subject: [PATCH 01/53] Added support to Debian OS --- Linux Script/configure-linux.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Linux Script/configure-linux.sh b/Linux Script/configure-linux.sh index 42ac9dd..8098e5b 100644 --- a/Linux Script/configure-linux.sh +++ b/Linux Script/configure-linux.sh @@ -15,7 +15,7 @@ function ctrl_c() { #name of the current script. This will get overwritten by the child script which calls this SCRIPT_NAME=configure-linux.sh #version of the current script. This will get overwritten by the child script which calls this -SCRIPT_VERSION=1.7 +SCRIPT_VERSION=1.8 #application tag. This will get overwritten by the child script which calls this APP_TAG= @@ -205,6 +205,9 @@ checkIfSupportedOS() *"centos"* ) echo "INFO: Operating system is CentOS." ;; + *"debian"* ) + echo "INFO: Operating system is Debian." + ;; *"amazon"* ) echo "INFO: Operating system is Amazon AMI." ;; From 7210c7d8fac011c824b9d067ede84f3ca22c7435 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Tue, 28 Oct 2014 18:04:21 +0530 Subject: [PATCH 02/53] Added a check for syslog-ng --- Linux Script/configure-linux.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Linux Script/configure-linux.sh b/Linux Script/configure-linux.sh index 8098e5b..c981418 100644 --- a/Linux Script/configure-linux.sh +++ b/Linux Script/configure-linux.sh @@ -15,7 +15,7 @@ function ctrl_c() { #name of the current script. This will get overwritten by the child script which calls this SCRIPT_NAME=configure-linux.sh #version of the current script. This will get overwritten by the child script which calls this -SCRIPT_VERSION=1.8 +SCRIPT_VERSION=1.9 #application tag. This will get overwritten by the child script which calls this APP_TAG= @@ -31,6 +31,8 @@ LOGGLY_RSYSLOG_CONFFILE_BACKUP=$LOGGLY_RSYSLOG_CONFFILE.loggly.bk RSYSLOG_DIR=/var/spool/rsyslog #rsyslog service name RSYSLOG_SERVICE=rsyslog +#syslog-ng +SYSLOG_NG_SERVICE=syslog-ng #rsyslogd RSYSLOGD=rsyslogd #minimum version of rsyslog to enable logging to loggly @@ -356,6 +358,7 @@ checkIfRsyslogConfiguredAsService() if [ -f /etc/init.d/$RSYSLOG_SERVICE ]; then logMsgToConfigSysLog "INFO" "INFO: $RSYSLOG_SERVICE is present as service." else + checkifSyslogNgConfiguredAsService logMsgToConfigSysLog "ERROR" "ERROR: $RSYSLOG_SERVICE is not present as service." exit 1 fi @@ -366,6 +369,13 @@ checkIfRsyslogConfiguredAsService() fi } +checkifSyslogNgConfiguredAsService() +{ + if [ -f /etc/init.d/$SYSLOG_NG_SERVICE ]; then + logMsgToConfigSysLog "ERROR" "ERROR: This script does not currently support syslog-ng. Please follow the instructions on this page https://www.loggly.com/docs/syslog-ng-manual-configuration" + exit 1 + fi +} #check if multiple versions of rsyslog is configured checkIfMultipleRsyslogConfigured() @@ -627,6 +637,7 @@ logMsgToConfigSysLog() #if it is a success, then log message "Script Succeeded" to config syslog and exit the script if [[ $cslStatus == "SUCCESS" ]]; then sendPayloadToConfigSysLog "SUCCESS" "Script Succeeded" "$enabler" + exit 0 fi } From b7826171d6fa26742a1a025fbd93d1b0321dc363 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Tue, 28 Oct 2014 18:06:01 +0530 Subject: [PATCH 03/53] Check if logs are parsed in Loggly --- Modular Scripts/Apache2/configure-apache.sh | 26 ++++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/Modular Scripts/Apache2/configure-apache.sh b/Modular Scripts/Apache2/configure-apache.sh index e1ccf12..6e99776 100644 --- a/Modular Scripts/Apache2/configure-apache.sh +++ b/Modular Scripts/Apache2/configure-apache.sh @@ -9,7 +9,7 @@ source configure-linux.sh "being-invoked" #name of the current script SCRIPT_NAME=configure-apache.sh #version of the current script -SCRIPT_VERSION=1.2 +SCRIPT_VERSION=1.3 #we have not found the apache version yet at this point in the script APP_TAG="\"apache-version\":\"\"" @@ -75,7 +75,7 @@ installLogglyConfForApache() checkIfApacheLogsMadeToLoggly #log success message - logMsgToConfigSysLog "SUCCESS" "SUCCESS: Apache successfully configured to send logs via Loggly." + logMsgToConfigSysLog "SUCCESS" "SUCCESS: Apache successfully configured to send logs to Loggly." } #executing script to remove loggly configuration for Apache @@ -297,14 +297,28 @@ checkIfApacheLogsMadeToLoggly() searchAndFetch apacheLatestLogCount "$queryUrl" let counter=$counter+1 if [ "$counter" -gt "$maxCounter" ]; then - logMsgToConfigSysLog "ERROR" "ERROR: Apache logs did not make to Loggly in time. Please check network and firewall settings and retry." - exit 1 + logMsgToConfigSysLog "ERROR" "ERROR: Apache logs did not make to Loggly in time. Please check network and firewall settings and retry." + exit 1 fi done if [ "$apacheLatestLogCount" -gt "$apacheInitialLogCount" ]; then - logMsgToConfigSysLog "SUCCESS" "SUCCESS: Apache logs successfully transferred to Loggly! You are now sending Apache logs to Loggly." - exit 0 + logMsgToConfigSysLog "INFO" "INFO: Apache logs successfully transferred to Loggly! You are now sending Apache logs to Loggly." + checkIfLogsAreParsedInLoggly + fi +} +#verifying if the logs are being parsed or not +checkIfLogsAreParsedInLoggly() +{ + apacheInitialLogCount=0 + queryParam="tag%3Aapache%20logtype%3Aapache&from=-15m&until=now&size=1" + queryUrl="$LOGGLY_ACCOUNT_URL/apiv2/search?q=$queryParam" + searchAndFetch apacheInitialLogCount "$queryUrl" + logMsgToConfigSysLog "INFO" "INFO: Verifying if the Apache logs are parsed in Loggly." + if [ "$apacheInitialLogCount" -gt 0 ]; then + logMsgToConfigSysLog "INFO" "INFO: Apache logs successfully parsed in Loggly!" + else + logMsgToConfigSysLog "WARN" "WARN: We received your logs but they do not appear to use one of our automatically parsed formats. You can still do full text search and counts on these logs, but you won't be able to use our field explorer. Please consider switching to one of our automated formats https://www.loggly.com/docs/automated-parsing/" fi } From ca64536f93702a3e19020c9d53eef27ebf9303a1 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Tue, 28 Oct 2014 18:07:11 +0530 Subject: [PATCH 04/53] Check if logs are parsed in Loggly --- .../configure-file-monitoring.sh | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/Modular Scripts/File Monitoring/configure-file-monitoring.sh b/Modular Scripts/File Monitoring/configure-file-monitoring.sh index 18f3fce..3fed65e 100644 --- a/Modular Scripts/File Monitoring/configure-file-monitoring.sh +++ b/Modular Scripts/File Monitoring/configure-file-monitoring.sh @@ -9,7 +9,7 @@ source configure-linux.sh "being-invoked" #name of the current script SCRIPT_NAME=configure-file-monitoring.sh #version of the current script -SCRIPT_VERSION=1.5 +SCRIPT_VERSION=1.6 #file to monitor (contains complete path and file name) provided by user LOGGLY_FILE_TO_MONITOR= @@ -77,9 +77,11 @@ installLogglyConfForFile() #verify if the file logs made it to loggly checkIfFileLogsMadeToLoggly - - #log success message - logMsgToConfigSysLog "SUCCESS" "SUCCESS: Successfully configured to send $LOGGLY_FILE_TO_MONITOR logs via Loggly." + + if [ "$IS_FILE_MONITOR_SCRIPT_INVOKED" = "false" ]; then + #log success message + logMsgToConfigSysLog "SUCCESS" "SUCCESS: Successfully configured to send $LOGGLY_FILE_TO_MONITOR logs via Loggly." + fi } #executing script to remove loggly configuration for File @@ -293,13 +295,23 @@ checkIfFileLogsMadeToLoggly() done if [ "$fileLatestLogCount" -gt "$fileInitialLogCount" ]; then - logMsgToConfigSysLog "SUCCESS" "SUCCESS: Logs successfully transferred to Loggly! You are now sending $LOGGLY_FILE_TO_MONITOR logs to Loggly." - if [ "$IS_FILE_MONITOR_SCRIPT_INVOKED" = "false" ]; then - exit 0 - fi + logMsgToConfigSysLog "INFO" "INFO: Logs successfully transferred to Loggly! You are now sending $LOGGLY_FILE_TO_MONITOR logs to Loggly." + checkIfLogsAreParsedInLoggly + fi +} +#verifying if the logs are being parsed or not +checkIfLogsAreParsedInLoggly() +{ + fileInitialLogCount=0 + queryParam="syslog.appName%3A$LOGGLY_FILE_TO_MONITOR_ALIAS%20tag%3A$LOGGLY_FILE_TAG&from=-15m&until=now&size=1" + queryUrl="$LOGGLY_ACCOUNT_URL/apiv2/search?q=$queryParam" + searchAndFetch fileInitialLogCount "$queryUrl" + if [ "$fileInitialLogCount" -gt 0 ]; then + logMsgToConfigSysLog "INFO" "INFO: File logs successfully parsed in Loggly!" + else + logMsgToConfigSysLog "WARN" "WARN: We received your logs but they do not appear to use one of our automatically parsed formats. You can still do full text search and counts on these logs, but you won't be able to use our field explorer. Please consider switching to one of our automated formats https://www.loggly.com/docs/automated-parsing/" fi } - #checks if the conf file exist. Name of conf file is constructed using the file alias name provided checkIfConfFileExist() { From c217c360f3701b9360a634f1104454bff6a239c3 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Tue, 28 Oct 2014 18:08:19 +0530 Subject: [PATCH 05/53] Check if logs are parsed in Loggly --- Modular Scripts/Nginx/configure-nginx.sh | 25 +++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/Modular Scripts/Nginx/configure-nginx.sh b/Modular Scripts/Nginx/configure-nginx.sh index 3df7157..6dbff67 100644 --- a/Modular Scripts/Nginx/configure-nginx.sh +++ b/Modular Scripts/Nginx/configure-nginx.sh @@ -9,7 +9,7 @@ source configure-linux.sh "being-invoked" #name of the current script SCRIPT_NAME=configure-nginx.sh #version of the current script -SCRIPT_VERSION=1.0 +SCRIPT_VERSION=1.1 #we have not found the nginx version yet at this point in the script APP_TAG="\"nginx-version\":\"\"" @@ -73,9 +73,10 @@ installLogglyConfForNginx() #verify if the nginx logs made it to loggly checkIfNginxLogsMadeToLoggly + #log success message - logMsgToConfigSysLog "SUCCESS" "SUCCESS: Nginx successfully configured to send logs via Loggly." + logMsgToConfigSysLog "SUCCESS" "SUCCESS: Nginx successfully configured to send logs to Loggly." } #executing script to remove loggly configuration for Nginx @@ -229,7 +230,6 @@ EOIPFW restartRsyslog } - #checks if the nginx logs made to loggly checkIfNginxLogsMadeToLoggly() { @@ -267,8 +267,23 @@ checkIfNginxLogsMadeToLoggly() done if [ "$nginxLatestLogCount" -gt "$nginxInitialLogCount" ]; then - logMsgToConfigSysLog "SUCCESS" "SUCCESS: nginx logs successfully transferred to Loggly! You are now sending nginx logs to Loggly." - exit 0 + logMsgToConfigSysLog "INFO" "INFO: Nginx logs successfully transferred to Loggly! You are now sending Nginx logs to Loggly." + checkIfLogsAreParsedInLoggly + fi +} + +#verifying if the logs are being parsed or not +checkIfLogsAreParsedInLoggly() +{ + nginxInitialLogCount=0 + queryParam="tag%3Anginx%20logtype%3Anginx&from=-15m&until=now&size=1" + queryUrl="$LOGGLY_ACCOUNT_URL/apiv2/search?q=$queryParam" + searchAndFetch nginxInitialLogCount "$queryUrl" + logMsgToConfigSysLog "INFO" "INFO: Verifying if the Nginx logs are parsed in Loggly." + if [ "$nginxInitialLogCount" -gt 0 ]; then + logMsgToConfigSysLog "INFO" "INFO: Nginx logs successfully parsed in Loggly!" + else + logMsgToConfigSysLog "WARN" "WARN: We received your logs but they do not appear to use one of our automatically parsed formats. You can still do full text search and counts on these logs, but you won't be able to use our field explorer. Please consider switching to one of our automated formats https://www.loggly.com/docs/automated-parsing/" fi } From c60835aa1d9a186d3a95309948f348d5d76b3fca Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Fri, 31 Oct 2014 18:22:35 +0530 Subject: [PATCH 06/53] Added support to suppress prompts A new argument '-s' is added to the script to support the 'No Prompt' mode. User can use this argument if he wants to give no inputs in the script at runtime. --- Linux Script/configure-linux.sh | 72 ++++++++++++++++++++------------- 1 file changed, 43 insertions(+), 29 deletions(-) diff --git a/Linux Script/configure-linux.sh b/Linux Script/configure-linux.sh index c981418..9a5980d 100644 --- a/Linux Script/configure-linux.sh +++ b/Linux Script/configure-linux.sh @@ -15,7 +15,7 @@ function ctrl_c() { #name of the current script. This will get overwritten by the child script which calls this SCRIPT_NAME=configure-linux.sh #version of the current script. This will get overwritten by the child script which calls this -SCRIPT_VERSION=1.9 +SCRIPT_VERSION=1.10 #application tag. This will get overwritten by the child script which calls this APP_TAG= @@ -69,6 +69,9 @@ LOGGLY_USERNAME= #this is a mandatory input LOGGLY_PASSWORD= +#if this variable is set to true then suppress all prompts +SUPPRESS_PROMPT="false" + #variables used in 22-loggly.conf file LOGGLY_SYSLOG_PORT=514 LOGGLY_DISTRIBUTION_ID="41058" @@ -220,17 +223,19 @@ checkIfSupportedOS() ;; * ) logMsgToConfigSysLog "WARN" "WARN: The linux distribution '$LINUX_DIST' has not been previously tested with Loggly." - while true; do - read -p "Would you like to continue anyway? (yes/no)" yn - case $yn in - [Yy]* ) - break;; - [Nn]* ) - exit 1 - ;; - * ) echo "Please answer yes or no.";; - esac - done + if [ "$SUPPRESS_PROMPT" == "false" ]; then + while true; do + read -p "Would you like to continue anyway? (yes/no)" yn + case $yn in + [Yy]* ) + break;; + [Nn]* ) + exit 1 + ;; + * ) echo "Please answer yes or no.";; + esac + done + fi ;; esac } @@ -456,22 +461,28 @@ checkIfConfigurationChanged() if [ "$ASK_FOR_VERIFICATION" == "true" ]; then logMsgToConfigSysLog "WARN" "WARN: Loggly rsyslog file /etc/rsyslog.d/22-loggly.conf content has changed." - while true; - do - read -p "Do you wish to override $LOGGLY_RSYSLOG_CONFFILE and re-verify configuration? (yes/no)" yn - case $yn in - [Yy]* ) - logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $LOGGLY_RSYSLOG_CONFFILE to $LOGGLY_RSYSLOG_CONFFILE_BACKUP"; - sudo mv -f $LOGGLY_RSYSLOG_CONFFILE $LOGGLY_RSYSLOG_CONFFILE_BACKUP; - checkAuthTokenAndWriteContents; - break;; - [Nn]* ) - LINUX_DO_VERIFICATION="false" - logMsgToConfigSysLog "INFO" "INFO: Skipping Linux verification." - break;; - * ) echo "Please answer yes or no.";; - esac - done + if [ "$SUPPRESS_PROMPT" == "false" ]; then + while true; + do + read -p "Do you wish to override $LOGGLY_RSYSLOG_CONFFILE and re-verify configuration? (yes/no)" yn + case $yn in + [Yy]* ) + logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $LOGGLY_RSYSLOG_CONFFILE to $LOGGLY_RSYSLOG_CONFFILE_BACKUP"; + sudo mv -f $LOGGLY_RSYSLOG_CONFFILE $LOGGLY_RSYSLOG_CONFFILE_BACKUP; + checkAuthTokenAndWriteContents; + break;; + [Nn]* ) + LINUX_DO_VERIFICATION="false" + logMsgToConfigSysLog "INFO" "INFO: Skipping Linux verification." + break;; + * ) echo "Please answer yes or no.";; + esac + done + else + logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $LOGGLY_RSYSLOG_CONFFILE to $LOGGLY_RSYSLOG_CONFFILE_BACKUP"; + sudo mv -f $LOGGLY_RSYSLOG_CONFFILE $LOGGLY_RSYSLOG_CONFFILE_BACKUP; + checkAuthTokenAndWriteContents; + fi else LINUX_DO_VERIFICATION="false" fi @@ -701,7 +712,7 @@ getPassword() usage() { cat << EOF -usage: configure-linux [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] +usage: configure-linux [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-s suppress prompts {optional)] usage: configure-linux [-a loggly auth account or subdomain] [-r to remove] usage: configure-linux [-h for help] EOF @@ -733,6 +744,9 @@ if [ "$1" != "being-invoked" ]; then -r | --remove ) LOGGLY_REMOVE="true" ;; + -s | --suppress ) + SUPPRESS_PROMPT="true" + ;; -h | --help) usage exit From c53de77219f6a03e639ca7177c21a91504cf6f3c Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Fri, 31 Oct 2014 18:23:53 +0530 Subject: [PATCH 07/53] Added support to suppress prompts A new argument '-s' is added to the script to support the 'No Prompt' mode. User can use this argument if he wants to give no inputs in the script at runtime. --- Modular Scripts/Apache2/configure-apache.sh | 70 ++++++++++++--------- 1 file changed, 42 insertions(+), 28 deletions(-) diff --git a/Modular Scripts/Apache2/configure-apache.sh b/Modular Scripts/Apache2/configure-apache.sh index 6e99776..b59c54b 100644 --- a/Modular Scripts/Apache2/configure-apache.sh +++ b/Modular Scripts/Apache2/configure-apache.sh @@ -9,7 +9,7 @@ source configure-linux.sh "being-invoked" #name of the current script SCRIPT_NAME=configure-apache.sh #version of the current script -SCRIPT_VERSION=1.3 +SCRIPT_VERSION=1.4 #we have not found the apache version yet at this point in the script APP_TAG="\"apache-version\":\"\"" @@ -172,20 +172,24 @@ checkLogFileSize() errorFileSize=$(wc -c "$2" | cut -f 1 -d ' ') fileSize=$((accessFileSize+errorFileSize)) if [ $fileSize -ge 102400000 ]; then - logMsgToConfigSysLog "INFO" "INFO: " - while true; do - read -p "WARN: There are currently large log files which may use up your allowed volume. Please rotate your logs before continuing. Would you like to continue now anyway? (yes/no)" yn - case $yn in - [Yy]* ) - logMsgToConfigSysLog "INFO" "INFO: Current apache logs size is $fileSize bytes. Continuing with Apache Loggly configuration."; - break;; - [Nn]* ) - logMsgToConfigSysLog "INFO" "INFO: Current apache logs size is $fileSize bytes. Discontinuing with Apache Loggly configuration." - exit 1 - break;; - * ) echo "Please answer yes or no.";; - esac - done + if [ "$SUPPRESS_PROMPT" == "false" ]; then + while true; do + read -p "WARN: There are currently large log files which may use up your allowed volume. Please rotate your logs before continuing. Would you like to continue now anyway? (yes/no)" yn + case $yn in + [Yy]* ) + logMsgToConfigSysLog "INFO" "INFO: Current apache logs size is $fileSize bytes. Continuing with Apache Loggly configuration."; + break;; + [Nn]* ) + logMsgToConfigSysLog "INFO" "INFO: Current apache logs size is $fileSize bytes. Discontinuing with Apache Loggly configuration." + exit 1 + break;; + * ) echo "Please answer yes or no.";; + esac + done + else + logMsgToConfigSysLog "WARN" "WARN: There are currently large log files which may use up your allowed volume." + logMsgToConfigSysLog "INFO" "INFO: Current apache logs size is $fileSize bytes. Continuing with Apache Loggly configuration." + fi elif [ $fileSize -eq 0 ]; then logMsgToConfigSysLog "WARN" "WARN: There are no recent logs from Apache there so won't be any sent to Loggly. You can generate some logs by visiting a page on your web server." exit 1 @@ -197,19 +201,26 @@ write21ApacheConfFile() #Create apache syslog config file if it doesn't exist echo "INFO: Checking if apache sysconf file $APACHE_SYSLOG_CONFFILE exist." if [ -f "$APACHE_SYSLOG_CONFFILE" ]; then + logMsgToConfigSysLog "WARN" "WARN: Apache syslog file $APACHE_SYSLOG_CONFFILE already exist." - while true; do - read -p "Do you wish to override $APACHE_SYSLOG_CONFFILE? (yes/no)" yn - case $yn in - [Yy]* ) - logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $APACHE_SYSLOG_CONFFILE to $APACHE_SYSLOG_CONFFILE_BACKUP"; - sudo mv -f $APACHE_SYSLOG_CONFFILE $APACHE_SYSLOG_CONFFILE_BACKUP; - write21ApacheFileContents; - break;; - [Nn]* ) break;; - * ) echo "Please answer yes or no.";; - esac - done + if [ "$SUPPRESS_PROMPT" == "false" ]; then + while true; do + read -p "Do you wish to override $APACHE_SYSLOG_CONFFILE? (yes/no)" yn + case $yn in + [Yy]* ) + logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $APACHE_SYSLOG_CONFFILE to $APACHE_SYSLOG_CONFFILE_BACKUP"; + sudo mv -f $APACHE_SYSLOG_CONFFILE $APACHE_SYSLOG_CONFFILE_BACKUP; + write21ApacheFileContents; + break;; + [Nn]* ) break;; + * ) echo "Please answer yes or no.";; + esac + done + else + logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $APACHE_SYSLOG_CONFFILE to $APACHE_SYSLOG_CONFFILE_BACKUP"; + sudo mv -f $APACHE_SYSLOG_CONFFILE $APACHE_SYSLOG_CONFFILE_BACKUP; + write21ApacheFileContents; + fi else write21ApacheFileContents fi @@ -337,7 +348,7 @@ remove21ApacheConfFile() usage() { cat << EOF -usage: configure-apache [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] +usage: configure-apache [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-s suppress prompts {optional)] usage: configure-apache [-a loggly auth account or subdomain] [-r to rollback] usage: configure-apache [-h for help] EOF @@ -369,6 +380,9 @@ while [ "$1" != "" ]; do -r | --rollback ) LOGGLY_ROLLBACK="true" ;; + -s | --suppress ) + SUPPRESS_PROMPT="true" + ;; -h | --help) usage exit From 995d96a4663b7c6a254dc2ed6f7ec25cb47c8eea Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Fri, 31 Oct 2014 18:24:54 +0530 Subject: [PATCH 08/53] Added support to suppress prompts A new argument '-s' is added to the script to support the 'No Prompt' mode. User can use this argument if he wants to give no inputs in the script at runtime. --- .../configure-file-monitoring.sh | 73 +++++++++++-------- 1 file changed, 42 insertions(+), 31 deletions(-) diff --git a/Modular Scripts/File Monitoring/configure-file-monitoring.sh b/Modular Scripts/File Monitoring/configure-file-monitoring.sh index 3fed65e..6e919bb 100644 --- a/Modular Scripts/File Monitoring/configure-file-monitoring.sh +++ b/Modular Scripts/File Monitoring/configure-file-monitoring.sh @@ -9,7 +9,7 @@ source configure-linux.sh "being-invoked" #name of the current script SCRIPT_NAME=configure-file-monitoring.sh #version of the current script -SCRIPT_VERSION=1.6 +SCRIPT_VERSION=1.7 #file to monitor (contains complete path and file name) provided by user LOGGLY_FILE_TO_MONITOR= @@ -149,20 +149,25 @@ checkIfFileAliasExist() { if [ -f "$FILE_SYSLOG_CONFFILE" ]; then logMsgToConfigSysLog "WARN" "WARN: This file alias is already taken. You must choose a unique file alias for each file." - while true; do - read -p "Would you like to overwrite the configuration for this file alias (yes/no)?" yn - case $yn in - [Yy]* ) - logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $FILE_SYSLOG_CONFFILE to $FILE_SYSLOG_CONFFILE_BACKUP"; - sudo mv -f $FILE_SYSLOG_CONFFILE $FILE_SYSLOG_CONFFILE_BACKUP; - break;; - [Nn]* ) - logMsgToConfigSysLog "INFO" "INFO: Not overwriting the existing configuration. Exiting" - exit 1 - break;; - * ) echo "Please answer yes or no.";; - esac - done + if [ "$SUPPRESS_PROMPT" == "false" ]; then + while true; do + read -p "Would you like to overwrite the configuration for this file alias (yes/no)?" yn + case $yn in + [Yy]* ) + logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $FILE_SYSLOG_CONFFILE to $FILE_SYSLOG_CONFFILE_BACKUP"; + sudo mv -f $FILE_SYSLOG_CONFFILE $FILE_SYSLOG_CONFFILE_BACKUP; + break;; + [Nn]* ) + logMsgToConfigSysLog "INFO" "INFO: Not overwriting the existing configuration. Exiting" + exit 1 + break;; + * ) echo "Please answer yes or no.";; + esac + done + else + logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $FILE_SYSLOG_CONFFILE to $FILE_SYSLOG_CONFFILE_BACKUP"; + sudo mv -f $FILE_SYSLOG_CONFFILE $FILE_SYSLOG_CONFFILE_BACKUP; + fi fi } @@ -172,20 +177,24 @@ checkLogFileSize() { monitorFileSize=$(wc -c "$1" | cut -f 1 -d ' ') if [ $monitorFileSize -ge 102400000 ]; then - logMsgToConfigSysLog "INFO" "INFO: " - while true; do - read -p "WARN: There are currently large log files which may use up your allowed volume. Please rotate your logs before continuing. Would you like to continue now anyway? (yes/no)" yn - case $yn in - [Yy]* ) - logMsgToConfigSysLog "INFO" "INFO: Current size of $LOGGLY_FILE_TO_MONITOR is $monitorFileSize bytes. Continuing with File Loggly configuration."; - break;; - [Nn]* ) - logMsgToConfigSysLog "INFO" "INFO: Current size of $LOGGLY_FILE_TO_MONITOR is $monitorFileSize bytes. Discontinuing with File Loggly configuration." - exit 1 - break;; - * ) echo "Please answer yes or no.";; - esac - done + if [ "$SUPPRESS_PROMPT" == "false" ]; then + while true; do + read -p "WARN: There are currently large log files which may use up your allowed volume. Please rotate your logs before continuing. Would you like to continue now anyway? (yes/no)" yn + case $yn in + [Yy]* ) + logMsgToConfigSysLog "INFO" "INFO: Current size of $LOGGLY_FILE_TO_MONITOR is $monitorFileSize bytes. Continuing with File Loggly configuration."; + break;; + [Nn]* ) + logMsgToConfigSysLog "INFO" "INFO: Current size of $LOGGLY_FILE_TO_MONITOR is $monitorFileSize bytes. Discontinuing with File Loggly configuration." + exit 1 + break;; + * ) echo "Please answer yes or no.";; + esac + done + else + logMsgToConfigSysLog "WARN" "WARN: There are currently large log files which may use up your allowed volume." + logMsgToConfigSysLog "INFO" "INFO: Current size of $LOGGLY_FILE_TO_MONITOR is $monitorFileSize bytes. Continuing with File Loggly configuration."; + fi elif [ $monitorFileSize -eq 0 ]; then logMsgToConfigSysLog "WARN" "WARN: There are no recent logs from $LOGGLY_FILE_TO_MONITOR so there won't be any data sent to Loggly. You can generate some logs by writing to this file." exit 1 @@ -200,7 +209,6 @@ checkFileReadPermission() { LINUX_DIST_IN_LOWER_CASE=$(echo $LINUX_DIST | tr "[:upper:]" "[:lower:]") - #no need to check read permissions with RedHat and CentOS as they also work with ---------- (000)permissions case "$LINUX_DIST_IN_LOWER_CASE" in *"redhat"* ) @@ -339,7 +347,7 @@ remove21ConfFile() usage() { cat << EOF -usage: configure-file-monitoring [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-f filename] [-tag filetag (optional)] [-l filealias] +usage: configure-file-monitoring [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-f filename] [-tag filetag (optional)] [-l filealias] [-s suppress prompts {optional)] usage: configure-file-monitoring [-a loggly auth account or subdomain] [-r to rollback] [-l filealias] usage: configure-file-monitoring [-h for help] EOF @@ -385,6 +393,9 @@ if [ "$1" != "being-invoked" ]; then LOGGLY_FILE_TAG=$1 echo "File tag: $LOGGLY_FILE_TAG" ;; + -s | --suppress ) + SUPPRESS_PROMPT="true" + ;; -h | --help) usage exit From ac4f3f14c89212cd25b802a5df687dba511fc9d1 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Fri, 31 Oct 2014 18:25:46 +0530 Subject: [PATCH 09/53] Added support to suppress prompts A new argument '-s' is added to the script to support the 'No Prompt' mode. User can use this argument if he wants to give no inputs in the script at runtime. --- Modular Scripts/Nginx/configure-nginx.sh | 71 ++++++++++++++---------- 1 file changed, 42 insertions(+), 29 deletions(-) diff --git a/Modular Scripts/Nginx/configure-nginx.sh b/Modular Scripts/Nginx/configure-nginx.sh index 6dbff67..ea007ac 100644 --- a/Modular Scripts/Nginx/configure-nginx.sh +++ b/Modular Scripts/Nginx/configure-nginx.sh @@ -9,7 +9,7 @@ source configure-linux.sh "being-invoked" #name of the current script SCRIPT_NAME=configure-nginx.sh #version of the current script -SCRIPT_VERSION=1.1 +SCRIPT_VERSION=1.2 #we have not found the nginx version yet at this point in the script APP_TAG="\"nginx-version\":\"\"" @@ -137,20 +137,24 @@ checkLogFileSize() errorFileSize=$(wc -c "$2" | cut -f 1 -d ' ') fileSize=$((accessFileSize+errorFileSize)) if [ $fileSize -ge 102400000 ]; then - logMsgToConfigSysLog "INFO" "INFO: " - while true; do - read -p "WARN: There are currently large log files which may use up your allowed volume. Please rotate your logs before continuing. Would you like to continue now anyway? (yes/no)" yn - case $yn in - [Yy]* ) - logMsgToConfigSysLog "INFO" "INFO: Current nginx logs size is $fileSize bytes. Continuing with nginx Loggly configuration."; - break;; - [Nn]* ) - logMsgToConfigSysLog "INFO" "INFO: Current nginx logs size is $fileSize bytes. Discontinuing with nginx Loggly configuration." - exit 1 - break;; - * ) echo "Please answer yes or no.";; - esac - done + if [ "$SUPPRESS_PROMPT" == "false" ]; then + while true; do + read -p "WARN: There are currently large log files which may use up your allowed volume. Please rotate your logs before continuing. Would you like to continue now anyway? (yes/no)" yn + case $yn in + [Yy]* ) + logMsgToConfigSysLog "INFO" "INFO: Current nginx logs size is $fileSize bytes. Continuing with nginx Loggly configuration."; + break;; + [Nn]* ) + logMsgToConfigSysLog "INFO" "INFO: Current nginx logs size is $fileSize bytes. Discontinuing with nginx Loggly configuration." + exit 1 + break;; + * ) echo "Please answer yes or no.";; + esac + done + else + logMsgToConfigSysLog "WARN" "WARN: There are currently large log files which may use up your allowed volume." + logMsgToConfigSysLog "INFO" "INFO: Current nginx logs size is $fileSize bytes. Continuing with nginx Loggly configuration."; + fi elif [ $fileSize -eq 0 ]; then logMsgToConfigSysLog "WARN" "WARN: There are no recent logs from nginx there so won't be any sent to Loggly. You can generate some logs by visiting a page on your web server." exit 1 @@ -162,19 +166,25 @@ write21NginxConfFile() #Create nginx syslog config file if it doesn't exist echo "INFO: Checking if nginx sysconf file $NGINX_SYSLOG_CONFFILE exist." if [ -f "$NGINX_SYSLOG_CONFFILE" ]; then - logMsgToConfigSysLog "WARN" "WARN: nginx syslog file $NGINX_SYSLOG_CONFFILE already exist." - while true; do - read -p "Do you wish to override $NGINX_SYSLOG_CONFFILE? (yes/no)" yn - case $yn in - [Yy]* ) - logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $NGINX_SYSLOG_CONFFILE to $NGINX_SYSLOG_CONFFILE_BACKUP"; - sudo mv -f $NGINX_SYSLOG_CONFFILE $NGINX_SYSLOG_CONFFILE_BACKUP; - write21NginxFileContents; - break;; - [Nn]* ) break;; - * ) echo "Please answer yes or no.";; - esac - done + logMsgToConfigSysLog "WARN" "WARN: nginx syslog file $NGINX_SYSLOG_CONFFILE already exist." + if [ "$SUPPRESS_PROMPT" == "false" ]; then + while true; do + read -p "Do you wish to override $NGINX_SYSLOG_CONFFILE? (yes/no)" yn + case $yn in + [Yy]* ) + logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $NGINX_SYSLOG_CONFFILE to $NGINX_SYSLOG_CONFFILE_BACKUP"; + sudo mv -f $NGINX_SYSLOG_CONFFILE $NGINX_SYSLOG_CONFFILE_BACKUP; + write21NginxFileContents; + break;; + [Nn]* ) break;; + * ) echo "Please answer yes or no.";; + esac + done + else + logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $NGINX_SYSLOG_CONFFILE to $NGINX_SYSLOG_CONFFILE_BACKUP"; + sudo mv -f $NGINX_SYSLOG_CONFFILE $NGINX_SYSLOG_CONFFILE_BACKUP; + write21NginxFileContents; + fi else write21NginxFileContents fi @@ -302,7 +312,7 @@ remove21NginxConfFile() usage() { cat << EOF -usage: configure-nginx [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] +usage: configure-nginx [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-s suppress prompts {optional)] usage: configure-nginx [-a loggly auth account or subdomain] [-r to rollback] usage: configure-nginx [-h for help] EOF @@ -334,6 +344,9 @@ while [ "$1" != "" ]; do -r | --rollback ) LOGGLY_ROLLBACK="true" ;; + -s | --suppress ) + SUPPRESS_PROMPT="true" + ;; -h | --help) usage exit From 8ea9e5577935e20134e92c4808bba3bcb57ea524 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Fri, 31 Oct 2014 18:27:28 +0530 Subject: [PATCH 10/53] Added support to suppress prompts A new argument '-s' is added to the script to support the 'No Prompt' mode. User can use this argument if he wants to give no inputs in the script at runtime. --- .../configure-s3-file-monitoring.sh | 154 ++++++++++-------- 1 file changed, 83 insertions(+), 71 deletions(-) diff --git a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh index f14d841..3a1bcde 100644 --- a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh +++ b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh @@ -8,7 +8,7 @@ source configure-file-monitoring.sh "being-invoked" #name of the current script SCRIPT_NAME=configure-s3-file-monitoring.sh #version of the current script -SCRIPT_VERSION=1.3 +SCRIPT_VERSION=1.4 #s3 bucket name to configure LOGGLY_S3_BUCKET_NAME= @@ -211,18 +211,20 @@ createTempDir() if [ -d "$TEMP_DIR" ]; then if [ "$(ls -A $TEMP_DIR)" ]; then logMsgToConfigSysLog "WARN" "WARN: There are some files/folders already present in $TEMP_DIR. If you continue, the files currently inside the $TEMP_DIR will also be configured to send logs to loggly." - while true; do - read -p "Would you like to continue now anyway? (yes/no)" yn - case $yn in - [Yy]* ) - break;; - [Nn]* ) - logMsgToConfigSysLog "INFO" "INFO: Discontinuing with s3 file monitoring configuration." - exit 1 - break;; - * ) echo "Please answer yes or no.";; - esac - done + if [ "$SUPPRESS_PROMPT" == "false" ]; then + while true; do + read -p "Would you like to continue now anyway? (yes/no)" yn + case $yn in + [Yy]* ) + break;; + [Nn]* ) + logMsgToConfigSysLog "INFO" "INFO: Discontinuing with s3 file monitoring configuration." + exit 1 + break;; + * ) echo "Please answer yes or no.";; + esac + done + fi fi else if [ -d "/tmp/s3monitoring" ]; then @@ -286,64 +288,71 @@ invokeS3FileMonitoring() installCronToSyncS3BucketPeriodically() { - while true; do - read -p "Would you like install a Cron job to sync the files currently in your bucket every 5 minutes? (yes/no)" yn - case $yn in - [Yy]* ) - - #copying .s3cfg file to /root so that it can be used by crontab for sync - if ! sudo test -f "/root/.s3cfg" ; then - sudo cp $HOME/.s3cfg /root - fi - - CRON_FILE="/tmp/s3monitoring/cron_$LOGGLY_S3_ALIAS" - CRON_SYNC_PATH="/tmp/s3monitoring/$LOGGLY_S3_ALIAS" - - #checking if the provided s3 path if of directory or file - IS_DIR="true" - BUCKET_URL_LAST_VALUE=$(echo ${LOGGLY_S3_BUCKET_NAME##*/}) - - if [ "$BUCKET_URL_LAST_VALUE" != "" ]; then - for fle in $(find $CRON_SYNC_PATH -name $BUCKET_URL_LAST_VALUE) - do - if [ -f $fle ]; then - IS_DIR="false" - break - fi - done - fi - - #adding file name to the sync folder as the bucket path is - #provided upto a file - if [ "$IS_DIR" == "false" ]; then - CRON_SYNC_PATH="$CRON_SYNC_PATH/$BUCKET_URL_LAST_VALUE" - fi - - logMsgToConfigSysLog "INFO" "INFO: Creating a Cron job to sync $LOGGLY_S3_BUCKET_NAME files to $CRON_SYNC_PATH in every five minutes." - - #setting up cron job - CRON_JOB_TO_SYNC_S3_BUCKET="*/5 * * * * s3cmd sync $LOGGLY_S3_BUCKET_NAME --preserve $CRON_SYNC_PATH" - - EXISTING_CRONS=$(sudo crontab -l 2>&1) - case $EXISTING_CRONS in - no*) - ;; - *) - echo "$EXISTING_CRONS" >> $CRON_FILE - ;; - esac - - echo "$CRON_JOB_TO_SYNC_S3_BUCKET" >> $CRON_FILE - sudo crontab $CRON_FILE - sudo rm -fr $CRON_FILE - break;; - [Nn]* ) - logMsgToConfigSysLog "INFO" "INFO: Skipping Cron installation." - break;; - * ) echo "Please answer yes or no.";; - esac - done + if [ "$SUPPRESS_PROMPT" == "false" ]; then + while true; do + read -p "Would you like install a Cron job to sync the files currently in your bucket every 5 minutes? (yes/no)" yn + case $yn in + [Yy]* ) + doCronInstallation + break;; + [Nn]* ) + logMsgToConfigSysLog "INFO" "INFO: Skipping Cron installation." + break;; + * ) echo "Please answer yes or no.";; + esac + done + else + doCronInstallation + fi +} + +doCronInstallation() +{ + #copying .s3cfg file to /root so that it can be used by crontab for sync + if ! sudo test -f "/root/.s3cfg" ; then + sudo cp $HOME/.s3cfg /root + fi + + CRON_FILE="/tmp/s3monitoring/cron_$LOGGLY_S3_ALIAS" + CRON_SYNC_PATH="/tmp/s3monitoring/$LOGGLY_S3_ALIAS" + + #checking if the provided s3 path if of directory or file + IS_DIR="true" + BUCKET_URL_LAST_VALUE=$(echo ${LOGGLY_S3_BUCKET_NAME##*/}) + + if [ "$BUCKET_URL_LAST_VALUE" != "" ]; then + for fle in $(find $CRON_SYNC_PATH -name $BUCKET_URL_LAST_VALUE) + do + if [ -f $fle ]; then + IS_DIR="false" + break + fi + done + fi + + #adding file name to the sync folder as the bucket path is + #provided upto a file + if [ "$IS_DIR" == "false" ]; then + CRON_SYNC_PATH="$CRON_SYNC_PATH/$BUCKET_URL_LAST_VALUE" + fi + + logMsgToConfigSysLog "INFO" "INFO: Creating a Cron job to sync $LOGGLY_S3_BUCKET_NAME files to $CRON_SYNC_PATH in every five minutes." + + #setting up cron job + CRON_JOB_TO_SYNC_S3_BUCKET="*/5 * * * * s3cmd sync $LOGGLY_S3_BUCKET_NAME --preserve $CRON_SYNC_PATH" + + EXISTING_CRONS=$(sudo crontab -l 2>&1) + case $EXISTING_CRONS in + no*) + ;; + *) + echo "$EXISTING_CRONS" >> $CRON_FILE + ;; + esac + echo "$CRON_JOB_TO_SYNC_S3_BUCKET" >> $CRON_FILE + sudo crontab $CRON_FILE + sudo rm -fr $CRON_FILE } deleteTempDir() @@ -429,7 +438,7 @@ deleteS3CronFromCrontab() usage() { cat << EOF -usage: configure-s3-file-monitoring [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-s3url s3bucketname ] [-s3l s3alias] +usage: configure-s3-file-monitoring [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-s3url s3bucketname ] [-s3l s3alias] [-s suppress prompts {optional)] usage: configure-s3-file-monitoring [-a loggly auth account or subdomain] [-r to rollback] [-s3l s3alias] usage: configure-s3-file-monitoring [-h for help] EOF @@ -470,6 +479,9 @@ while [ "$1" != "" ]; do LOGGLY_S3_ALIAS=$1 echo "File alias: $LOGGLY_S3_ALIAS" ;; + -s | --suppress ) + SUPPRESS_PROMPT="true" + ;; -h | --help) usage exit From 6e316adae50e10a4a22e235e94b5eb46ee2418b7 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Fri, 31 Oct 2014 18:28:22 +0530 Subject: [PATCH 11/53] Added support to suppress prompts A new argument '-s' is added to the script to support the 'No Prompt' mode. User can use this argument if he wants to give no inputs in the script at runtime. --- Modular Scripts/Tomcat/configure-tomcat.sh | 65 +++++++++++++--------- 1 file changed, 39 insertions(+), 26 deletions(-) diff --git a/Modular Scripts/Tomcat/configure-tomcat.sh b/Modular Scripts/Tomcat/configure-tomcat.sh index 91f92ea..753c905 100644 --- a/Modular Scripts/Tomcat/configure-tomcat.sh +++ b/Modular Scripts/Tomcat/configure-tomcat.sh @@ -9,7 +9,7 @@ source configure-linux.sh "being-invoked" #name of the current script SCRIPT_NAME=configure-tomcat.sh #version of the current script -SCRIPT_VERSION=1.1 +SCRIPT_VERSION=1.2 #minimum version of tomcat to enable log rotation MIN_TOMCAT_VERSION=6.0.33.0 @@ -334,18 +334,22 @@ checkIfTomcatConfiguredWithLog4J() canTomcatBeRestarted() { - while true; do - read -p "Tomcat needs to be restarted during configuration. Do you wish to continue? (yes/no)" yn - case $yn in - [Yy]* ) - break;; - [Nn]* ) - logMsgToConfigSysLog "WARN" "WARN: This script must restart Tomcat. Please run the script again when you are ready to restart it. No changes have been made to your system. Exiting." - exit 1 - break;; - * ) echo "Please answer yes or no.";; - esac - done + if [ "$SUPPRESS_PROMPT" == "false" ]; then + while true; do + read -p "Tomcat needs to be restarted during configuration. Do you wish to continue? (yes/no)" yn + case $yn in + [Yy]* ) + break;; + [Nn]* ) + logMsgToConfigSysLog "WARN" "WARN: This script must restart Tomcat. Please run the script again when you are ready to restart it. No changes have been made to your system. Exiting." + exit 1 + break;; + * ) echo "Please answer yes or no.";; + esac + done + else + logMsgToConfigSysLog "WARN" "WARN:Tomcat needs to be restarted during configuration." + fi } #backup the logging.properties file in the CATALINA_HOME folder backupLoggingPropertiesFile() @@ -420,18 +424,24 @@ write21TomcatConfFile() echo "INFO: Checking if tomcat sysconf file $TOMCAT_SYSLOG_CONFFILE exist." if [ -f "$TOMCAT_SYSLOG_CONFFILE" ]; then logMsgToConfigSysLog "WARN" "WARN: Tomcat syslog file $TOMCAT_SYSLOG_CONFFILE already exist." - while true; do - read -p "Do you wish to override $TOMCAT_SYSLOG_CONFFILE? (yes/no)" yn - case $yn in - [Yy]* ) - logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $TOMCAT_SYSLOG_CONFFILE to $TOMCAT_SYSLOG_CONFFILE_BACKUP"; - sudo mv -f $TOMCAT_SYSLOG_CONFFILE $TOMCAT_SYSLOG_CONFFILE_BACKUP; - write21TomcatFileContents; - break;; - [Nn]* ) break;; - * ) echo "Please answer yes or no.";; - esac - done + if [ "$SUPPRESS_PROMPT" == "false" ]; then + while true; do + read -p "Do you wish to override $TOMCAT_SYSLOG_CONFFILE? (yes/no)" yn + case $yn in + [Yy]* ) + logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $TOMCAT_SYSLOG_CONFFILE to $TOMCAT_SYSLOG_CONFFILE_BACKUP"; + sudo mv -f $TOMCAT_SYSLOG_CONFFILE $TOMCAT_SYSLOG_CONFFILE_BACKUP; + write21TomcatFileContents; + break;; + [Nn]* ) break;; + * ) echo "Please answer yes or no.";; + esac + done + else + logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $TOMCAT_SYSLOG_CONFFILE to $TOMCAT_SYSLOG_CONFFILE_BACKUP"; + sudo mv -f $TOMCAT_SYSLOG_CONFFILE $TOMCAT_SYSLOG_CONFFILE_BACKUP; + write21TomcatFileContents; + fi else write21TomcatFileContents fi @@ -643,7 +653,7 @@ restartTomcat() usage() { cat << EOF -usage: configure-tomcat [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-ch catalina home (optional)] +usage: configure-tomcat [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-ch catalina home (optional)] [-s suppress prompts {optional)] usage: configure-tomcat [-r to rollback] [-a loggly auth account or subdomain] [-ch catalina home (optional)] usage: configure-tomcat [-h for help] EOF @@ -679,6 +689,9 @@ while [ "$1" != "" ]; do -r | --rollback ) LOGGLY_ROLLBACK="true" ;; + -s | --suppress ) + SUPPRESS_PROMPT="true" + ;; -h | --help) usage exit From 32cb0b8dda92732308b3b181e3b27017d4bf1fdf Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Fri, 31 Oct 2014 23:48:30 +0530 Subject: [PATCH 12/53] Linux Script: Minor Bug Fix --- Linux Script/configure-linux.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Linux Script/configure-linux.sh b/Linux Script/configure-linux.sh index 9a5980d..05ef750 100644 --- a/Linux Script/configure-linux.sh +++ b/Linux Script/configure-linux.sh @@ -498,7 +498,7 @@ inputStr=" # ------------------------------------------------------- # Define the template used for sending logs to Loggly. Do not change this format. -\$template LogglyFormat,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$2@$3] %msg%\" +\$template LogglyFormat,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$2@$3] %msg%\n\" # Send messages to Loggly over TCP using the template. *.* @@$4:$5;LogglyFormat From 743fc92b3bbc3b66555f7447d90b2eb9129f14e4 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Wed, 5 Nov 2014 00:12:11 +0530 Subject: [PATCH 13/53] Linux Script: Bug Fix --- Linux Script/configure-linux.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Linux Script/configure-linux.sh b/Linux Script/configure-linux.sh index 05ef750..41bf359 100644 --- a/Linux Script/configure-linux.sh +++ b/Linux Script/configure-linux.sh @@ -448,7 +448,7 @@ checkIfConfigurationChanged() #strings to be checked which should be present in the existing 22-loggly.conf. #If these strings are not same then a warning message will be shown to user to update the 22-loggly.conf file - STR_TO_BE_CHECKED[0]="\$template LogglyFormat,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@$LOGGLY_DISTRIBUTION_ID] %msg%\"" + STR_TO_BE_CHECKED[0]="\$template LogglyFormat,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@$LOGGLY_DISTRIBUTION_ID] %msg%\n\"" STR_TO_BE_CHECKED[1]="*.* @@$LOGS_01_HOST:$LOGGLY_SYSLOG_PORT;LogglyFormat" for i in "${STR_TO_BE_CHECKED[@]}" From 48e750baf6b2df925d47fc2e043ded720da85471 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Thu, 6 Nov 2014 23:35:51 +0530 Subject: [PATCH 14/53] File Monitoring: Check if logs are rotated. --- .../File Monitoring/configure-file-monitoring.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/Modular Scripts/File Monitoring/configure-file-monitoring.sh b/Modular Scripts/File Monitoring/configure-file-monitoring.sh index 6e919bb..6a8b9d3 100644 --- a/Modular Scripts/File Monitoring/configure-file-monitoring.sh +++ b/Modular Scripts/File Monitoring/configure-file-monitoring.sh @@ -9,7 +9,7 @@ source configure-linux.sh "being-invoked" #name of the current script SCRIPT_NAME=configure-file-monitoring.sh #version of the current script -SCRIPT_VERSION=1.7 +SCRIPT_VERSION=1.8 #file to monitor (contains complete path and file name) provided by user LOGGLY_FILE_TO_MONITOR= @@ -56,6 +56,9 @@ installLogglyConfForFile() #check if file to monitor exists checkIfFileExist + + #check if logrotation enabled + checkIfLogRotationEnabled #checks if the file has proper read permission checkFileReadPermission @@ -144,6 +147,15 @@ checkIfFileExist() fi } +#checks if log rotation is enabled on the selected file +checkIfLogRotationEnabled() +{ + FILENAME="${LOGGLY_FILE_TO_MONITOR##*/}" + if [ -f "/etc/logrotate.d/$FILENAME" ]; then + logMsgToConfigSysLog "WARN" "WARN: Log rotation is enabled on $FILENAME." + fi +} + #check if the file alias is already taken checkIfFileAliasExist() { From 1afae108b941add48697536c56c9295352fa75c4 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Tue, 11 Nov 2014 21:30:26 +0530 Subject: [PATCH 15/53] File Monitoring: Check if logs are rotated. Using grep to check if the file is mentioned in any of the config file present in the /etc/logrotate.d --- Modular Scripts/File Monitoring/configure-file-monitoring.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Modular Scripts/File Monitoring/configure-file-monitoring.sh b/Modular Scripts/File Monitoring/configure-file-monitoring.sh index 6a8b9d3..321b71c 100644 --- a/Modular Scripts/File Monitoring/configure-file-monitoring.sh +++ b/Modular Scripts/File Monitoring/configure-file-monitoring.sh @@ -150,9 +150,8 @@ checkIfFileExist() #checks if log rotation is enabled on the selected file checkIfLogRotationEnabled() { - FILENAME="${LOGGLY_FILE_TO_MONITOR##*/}" - if [ -f "/etc/logrotate.d/$FILENAME" ]; then - logMsgToConfigSysLog "WARN" "WARN: Log rotation is enabled on $FILENAME." + if [ $(grep -r "$LOGGLY_FILE_TO_MONITOR" /etc/logrotate.d/) ]; then + logMsgToConfigSysLog "WARN" "WARN: Log rotation is enabled on $LOGGLY_FILE_TO_MONITOR." fi } From bdddbb077023ef6c3ea69e47b333076ece49e43c Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Wed, 12 Nov 2014 21:12:50 +0530 Subject: [PATCH 16/53] Updated Mac logs link --- Linux Script/configure-linux.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Linux Script/configure-linux.sh b/Linux Script/configure-linux.sh index 41bf359..60e86fe 100644 --- a/Linux Script/configure-linux.sh +++ b/Linux Script/configure-linux.sh @@ -218,7 +218,7 @@ checkIfSupportedOS() ;; *"darwin"* ) #if the OS is mac then exit - logMsgToConfigSysLog "ERROR" "ERROR: This script is for Linux systems, and Darwin or Mac OSX are not currently supported. You can find alternative options here: https://www.loggly.com/docs" + logMsgToConfigSysLog "ERROR" "ERROR: This script is for Linux systems, and Darwin or Mac OSX are not currently supported. You can find alternative options here: https://www.loggly.com/docs/send-mac-logs-to-loggly/" exit 1 ;; * ) From ee39dc673fdfba4649d46b4742ee7b78a5fc6158 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Thu, 20 Nov 2014 23:16:23 +0530 Subject: [PATCH 17/53] Checks if log rotation is enabled on the File --- Modular Scripts/File Monitoring/configure-file-monitoring.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Modular Scripts/File Monitoring/configure-file-monitoring.sh b/Modular Scripts/File Monitoring/configure-file-monitoring.sh index 321b71c..dc4e210 100644 --- a/Modular Scripts/File Monitoring/configure-file-monitoring.sh +++ b/Modular Scripts/File Monitoring/configure-file-monitoring.sh @@ -150,7 +150,8 @@ checkIfFileExist() #checks if log rotation is enabled on the selected file checkIfLogRotationEnabled() { - if [ $(grep -r "$LOGGLY_FILE_TO_MONITOR" /etc/logrotate.d/) ]; then + FILENAME="${LOGGLY_FILE_TO_MONITOR%.*}" + if [[ $(grep -r "$FILENAME" /etc/logrotate.d/) ]]; then logMsgToConfigSysLog "WARN" "WARN: Log rotation is enabled on $LOGGLY_FILE_TO_MONITOR." fi } From 76af2ad406dd8a330688fc73a58295d6937703ab Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Thu, 20 Nov 2014 23:22:54 +0530 Subject: [PATCH 18/53] Checks if log rotation is enabled on the Log files --- Modular Scripts/Apache2/configure-apache.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Modular Scripts/Apache2/configure-apache.sh b/Modular Scripts/Apache2/configure-apache.sh index b59c54b..0129e59 100644 --- a/Modular Scripts/Apache2/configure-apache.sh +++ b/Modular Scripts/Apache2/configure-apache.sh @@ -117,6 +117,8 @@ checkApacheDetails() #set all the required apache variables by this script setApacheVariables + + checkIfLogRotationEnabled } #Get the apache service name on various linux flavors @@ -144,6 +146,15 @@ setApacheVariables() LOGGLY_APACHE_LOG_HOME=/var/log/$SERVICE } + +#checks if log rotation is enabled on the selected file +checkIfLogRotationEnabled() +{ + if [[ $(grep -r "/var/log/$SERVICE/*." /etc/logrotate.d/$SERVICE) ]]; then + logMsgToConfigSysLog "WARN" "WARN: Log rotation is enabled on $LOGGLY_APACHE_LOG_HOME/$APACHE_ACCESS_LOG_FILE and $LOGGLY_APACHE_LOG_HOME/$APACHE_ERROR_LOG_FILE." + fi +} + #gets the version of apache installed on the unix box getApacheVersion() { From 65e3be8cfef9d63d7d219a07b39b2eb4b23da54c Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Thu, 20 Nov 2014 23:30:06 +0530 Subject: [PATCH 19/53] Checks if log rotation is enabled on the Log files --- Modular Scripts/Nginx/configure-nginx.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Modular Scripts/Nginx/configure-nginx.sh b/Modular Scripts/Nginx/configure-nginx.sh index ea007ac..5fba6ef 100644 --- a/Modular Scripts/Nginx/configure-nginx.sh +++ b/Modular Scripts/Nginx/configure-nginx.sh @@ -99,6 +99,14 @@ removeLogglyConfForNginx() logMsgToConfigSysLog "INFO" "INFO: Rollback completed." } +#checks if log rotation is enabled on the selected file +checkIfLogRotationEnabled() +{ + if [[ $(grep -r "/var/log/$SERVICE/*." /etc/logrotate.d/$SERVICE) ]]; then + logMsgToConfigSysLog "WARN" "WARN: Log rotation is enabled on $LOGGLY_NGINX_LOG_HOME/$NGINX_ACCESS_LOG_FILE and $LOGGLY_NGINX_LOG_HOME/$NGINX_ERROR_LOG_FILE." + fi +} + #identify if nginx is installed on your system and is available as a service checkNginxDetails() { @@ -113,6 +121,8 @@ checkNginxDetails() #set all the required nginx variables by this script setNginxVariables + + checkIfLogRotationEnabled } From 8aff0e5a1eb9afc82f068033ef8828ea3c97fcb4 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Sat, 22 Nov 2014 00:08:09 +0530 Subject: [PATCH 20/53] Minor modifications --- Modular Scripts/File Monitoring/configure-file-monitoring.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modular Scripts/File Monitoring/configure-file-monitoring.sh b/Modular Scripts/File Monitoring/configure-file-monitoring.sh index dc4e210..b603d37 100644 --- a/Modular Scripts/File Monitoring/configure-file-monitoring.sh +++ b/Modular Scripts/File Monitoring/configure-file-monitoring.sh @@ -152,7 +152,7 @@ checkIfLogRotationEnabled() { FILENAME="${LOGGLY_FILE_TO_MONITOR%.*}" if [[ $(grep -r "$FILENAME" /etc/logrotate.d/) ]]; then - logMsgToConfigSysLog "WARN" "WARN: Log rotation is enabled on $LOGGLY_FILE_TO_MONITOR." + logMsgToConfigSysLog "WARN" "WARN: Log rotation is enabled on $LOGGLY_FILE_TO_MONITOR. Please follow instructions here to update logrotate https://www.loggly.com/docs/log-rotate" fi } From 9d3f56173b23d618c6cbb40e89789c4e3edba607 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Sat, 22 Nov 2014 00:09:38 +0530 Subject: [PATCH 21/53] Minor modifications --- Modular Scripts/Apache2/configure-apache.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modular Scripts/Apache2/configure-apache.sh b/Modular Scripts/Apache2/configure-apache.sh index 0129e59..a4989c0 100644 --- a/Modular Scripts/Apache2/configure-apache.sh +++ b/Modular Scripts/Apache2/configure-apache.sh @@ -151,7 +151,7 @@ setApacheVariables() checkIfLogRotationEnabled() { if [[ $(grep -r "/var/log/$SERVICE/*." /etc/logrotate.d/$SERVICE) ]]; then - logMsgToConfigSysLog "WARN" "WARN: Log rotation is enabled on $LOGGLY_APACHE_LOG_HOME/$APACHE_ACCESS_LOG_FILE and $LOGGLY_APACHE_LOG_HOME/$APACHE_ERROR_LOG_FILE." + logMsgToConfigSysLog "WARN" "WARN: Log rotation is enabled on $LOGGLY_APACHE_LOG_HOME/$APACHE_ACCESS_LOG_FILE and $LOGGLY_APACHE_LOG_HOME/$APACHE_ERROR_LOG_FILE. Please follow instructions here to update logrotate https://www.loggly.com/docs/log-rotate" fi } From 25754be37363e0915783a59f51f392e9d9ebb9a9 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Sat, 22 Nov 2014 00:23:06 +0530 Subject: [PATCH 22/53] Minor modifications --- Modular Scripts/Nginx/configure-nginx.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modular Scripts/Nginx/configure-nginx.sh b/Modular Scripts/Nginx/configure-nginx.sh index 5fba6ef..b51bcb9 100644 --- a/Modular Scripts/Nginx/configure-nginx.sh +++ b/Modular Scripts/Nginx/configure-nginx.sh @@ -103,7 +103,7 @@ removeLogglyConfForNginx() checkIfLogRotationEnabled() { if [[ $(grep -r "/var/log/$SERVICE/*." /etc/logrotate.d/$SERVICE) ]]; then - logMsgToConfigSysLog "WARN" "WARN: Log rotation is enabled on $LOGGLY_NGINX_LOG_HOME/$NGINX_ACCESS_LOG_FILE and $LOGGLY_NGINX_LOG_HOME/$NGINX_ERROR_LOG_FILE." + logMsgToConfigSysLog "WARN" "WARN: Log rotation is enabled on $LOGGLY_NGINX_LOG_HOME/$NGINX_ACCESS_LOG_FILE and $LOGGLY_NGINX_LOG_HOME/$NGINX_ERROR_LOG_FILE. Please follow instructions here to update logrotate https://www.loggly.com/docs/log-rotate" fi } From 33d993d0eba3b65c3d4d4c5f76f6f6d363e3cd75 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Tue, 25 Nov 2014 23:40:13 +0530 Subject: [PATCH 23/53] Added support to multiple tags --- .../configure-file-monitoring.sh | 36 +++++++++++++++---- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/Modular Scripts/File Monitoring/configure-file-monitoring.sh b/Modular Scripts/File Monitoring/configure-file-monitoring.sh index b603d37..1f95e16 100644 --- a/Modular Scripts/File Monitoring/configure-file-monitoring.sh +++ b/Modular Scripts/File Monitoring/configure-file-monitoring.sh @@ -9,7 +9,7 @@ source configure-linux.sh "being-invoked" #name of the current script SCRIPT_NAME=configure-file-monitoring.sh #version of the current script -SCRIPT_VERSION=1.8 +SCRIPT_VERSION=1.9 #file to monitor (contains complete path and file name) provided by user LOGGLY_FILE_TO_MONITOR= @@ -37,6 +37,9 @@ LOGGLY_FILE_TAG="file" #format name for the conf file. Can be set by calling script CONF_FILE_FORMAT_NAME="LogglyFormatFile" +#add tags to the logs +TAG= + ########## Variable Declarations - End ########## # executing the script for loggly to install and configure syslog @@ -68,7 +71,10 @@ installLogglyConfForFile() #configure loggly for Linux installLogglyConf - + + #multiple tags + addTagsInConfiguration + #create 21.conf file write21ConfFileContents @@ -239,6 +245,17 @@ checkFileReadPermission() } + +addTagsInConfiguration() +{ + #split tags by comman(,) + IFS=, read -a array <<< "$LOGGLY_FILE_TAG" + for i in "${array[@]}" + do + TAG="$TAG tag=\\\"$i\\\" " + done +} + #function to write the contents of syslog config file write21ConfFileContents() { @@ -254,7 +271,7 @@ write21ConfFileContents() imfileStr+="\$PrivDropToGroup adm " fi - + imfileStr+=" # File access file: \$InputFileName $LOGGLY_FILE_TO_MONITOR @@ -265,7 +282,7 @@ write21ConfFileContents() \$InputRunFileMonitor #Add a tag for file events - \$template $CONF_FILE_FORMAT_NAME,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 tag=\\\"$LOGGLY_FILE_TAG\\\"] %msg%\n\" + \$template $CONF_FILE_FORMAT_NAME,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 $TAG] %msg%\n\" if \$programname == '$LOGGLY_FILE_TO_MONITOR_ALIAS' then @@logs-01.loggly.com:514;$CONF_FILE_FORMAT_NAME if \$programname == '$LOGGLY_FILE_TO_MONITOR_ALIAS' then ~ @@ -323,7 +340,14 @@ checkIfFileLogsMadeToLoggly() checkIfLogsAreParsedInLoggly() { fileInitialLogCount=0 - queryParam="syslog.appName%3A$LOGGLY_FILE_TO_MONITOR_ALIAS%20tag%3A$LOGGLY_FILE_TAG&from=-15m&until=now&size=1" + TAG_PARSER= + IFS=, read -a array <<< "$LOGGLY_FILE_TAG" + for i in "${array[@]}" + do + TAG_PARSER="$TAG_PARSER%20tag%3A$i " + done + + queryParam="syslog.appName%3A$LOGGLY_FILE_TO_MONITOR_ALIAS$TAG_PARSER&from=-15m&until=now&size=1" queryUrl="$LOGGLY_ACCOUNT_URL/apiv2/search?q=$queryParam" searchAndFetch fileInitialLogCount "$queryUrl" if [ "$fileInitialLogCount" -gt 0 ]; then @@ -359,7 +383,7 @@ remove21ConfFile() usage() { cat << EOF -usage: configure-file-monitoring [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-f filename] [-tag filetag (optional)] [-l filealias] [-s suppress prompts {optional)] +usage: configure-file-monitoring [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-f filename] [-tag filetag1,filetag2 (optional)] [-l filealias] [-s suppress prompts {optional)] usage: configure-file-monitoring [-a loggly auth account or subdomain] [-r to rollback] [-l filealias] usage: configure-file-monitoring [-h for help] EOF From 75c172c26efd5639995398db49211b33fcda1d8a Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Tue, 25 Nov 2014 23:53:38 +0530 Subject: [PATCH 24/53] Added support to multiple tags --- Modular Scripts/Apache2/configure-apache.sh | 40 ++++++++++++++++++--- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/Modular Scripts/Apache2/configure-apache.sh b/Modular Scripts/Apache2/configure-apache.sh index a4989c0..de6975a 100644 --- a/Modular Scripts/Apache2/configure-apache.sh +++ b/Modular Scripts/Apache2/configure-apache.sh @@ -9,7 +9,7 @@ source configure-linux.sh "being-invoked" #name of the current script SCRIPT_NAME=configure-apache.sh #version of the current script -SCRIPT_VERSION=1.4 +SCRIPT_VERSION=1.5 #we have not found the apache version yet at this point in the script APP_TAG="\"apache-version\":\"\"" @@ -36,6 +36,12 @@ MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure Apache2 is available #this variable will hold if the check env function for linux is invoked APACHE_ENV_VALIDATED="false" + +#apache as tag sent with the logs +LOGGLY_FILE_TAG="apache" + +#add tags to the logs +TAG= ########## Variable Declarations - End ########## #check if apache environment is compatible for Loggly @@ -64,7 +70,10 @@ installLogglyConfForApache() #configure loggly for Linux installLogglyConf - + + #multiple tags + addTagsInConfiguration + #create 21apache.conf file write21ApacheConfFile @@ -118,6 +127,7 @@ checkApacheDetails() #set all the required apache variables by this script setApacheVariables + #to check logrotation checkIfLogRotationEnabled } @@ -237,6 +247,15 @@ write21ApacheConfFile() fi } +addTagsInConfiguration() +{ + #split tags by comman(,) + IFS=, read -a array <<< "$LOGGLY_FILE_TAG" + for i in "${array[@]}" + do + TAG="$TAG tag=\\\"$i\\\" " + done +} #function to write the contents of apache syslog config file write21ApacheFileContents() { @@ -271,7 +290,7 @@ write21ApacheFileContents() \$InputRunFileMonitor #Add a tag for apache events - \$template LogglyFormatApache,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 tag=\\\"apache\\\"] %msg%\n\" + \$template LogglyFormatApache,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 $TAG] %msg%\n\" if \$programname == 'apache-access' then @@logs-01.loggly.com:514;LogglyFormatApache if \$programname == 'apache-access' then ~ @@ -333,7 +352,14 @@ checkIfApacheLogsMadeToLoggly() checkIfLogsAreParsedInLoggly() { apacheInitialLogCount=0 - queryParam="tag%3Aapache%20logtype%3Aapache&from=-15m&until=now&size=1" + TAG_PARSER= + IFS=, read -a array <<< "$LOGGLY_FILE_TAG" + + for i in "${array[@]}" + do + TAG_PARSER="$TAG_PARSER%20tag%3A$i " + done + queryParam="logtype%3Aapache$TAG_PARSER&from=-15m&until=now&size=1" queryUrl="$LOGGLY_ACCOUNT_URL/apiv2/search?q=$queryParam" searchAndFetch apacheInitialLogCount "$queryUrl" logMsgToConfigSysLog "INFO" "INFO: Verifying if the Apache logs are parsed in Loggly." @@ -359,7 +385,7 @@ remove21ApacheConfFile() usage() { cat << EOF -usage: configure-apache [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-s suppress prompts {optional)] +usage: configure-apache [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-tag filetag1,filetag2 (optional)] [-s suppress prompts {optional)] usage: configure-apache [-a loggly auth account or subdomain] [-r to rollback] usage: configure-apache [-h for help] EOF @@ -388,6 +414,10 @@ while [ "$1" != "" ]; do -p | --password ) shift LOGGLY_PASSWORD=$1 ;; + -tag| --filetag ) shift + LOGGLY_FILE_TAG=$1 + echo "File tag: $LOGGLY_FILE_TAG" + ;; -r | --rollback ) LOGGLY_ROLLBACK="true" ;; From f8a92a1d0a5b2a2d6564afde82096ce8baa9d8f2 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Tue, 25 Nov 2014 23:59:15 +0530 Subject: [PATCH 25/53] Added support to multiple tags --- Modular Scripts/Nginx/configure-nginx.sh | 39 +++++++++++++++++++++--- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/Modular Scripts/Nginx/configure-nginx.sh b/Modular Scripts/Nginx/configure-nginx.sh index b51bcb9..44ad6a2 100644 --- a/Modular Scripts/Nginx/configure-nginx.sh +++ b/Modular Scripts/Nginx/configure-nginx.sh @@ -9,7 +9,7 @@ source configure-linux.sh "being-invoked" #name of the current script SCRIPT_NAME=configure-nginx.sh #version of the current script -SCRIPT_VERSION=1.2 +SCRIPT_VERSION=1.3 #we have not found the nginx version yet at this point in the script APP_TAG="\"nginx-version\":\"\"" @@ -36,6 +36,13 @@ MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure nginx is available a #this variable will hold if the check env function for linux is invoked NGINX_ENV_VALIDATED="false" + +#apache as tag sent with the logs +LOGGLY_FILE_TAG="nginx" + +#add tags to the logs +TAG= + ########## Variable Declarations - End ########## #check if nginx environment is compatible for Loggly @@ -65,6 +72,9 @@ installLogglyConfForNginx() #configure loggly for Linux installLogglyConf + #multiple tags + addTagsInConfiguration + #create 21nginx.conf file write21NginxConfFile @@ -74,7 +84,6 @@ installLogglyConfForNginx() #verify if the nginx logs made it to loggly checkIfNginxLogsMadeToLoggly - #log success message logMsgToConfigSysLog "SUCCESS" "SUCCESS: Nginx successfully configured to send logs to Loggly." } @@ -122,6 +131,7 @@ checkNginxDetails() #set all the required nginx variables by this script setNginxVariables + #to check logrotation checkIfLogRotationEnabled } @@ -200,6 +210,15 @@ write21NginxConfFile() fi } +addTagsInConfiguration() +{ + #split tags by comman(,) + IFS=, read -a array <<< "$LOGGLY_FILE_TAG" + for i in "${array[@]}" + do + TAG="$TAG tag=\\\"$i\\\" " + done +} #function to write the contents of nginx syslog config file write21NginxFileContents() { @@ -234,7 +253,7 @@ write21NginxFileContents() \$InputRunFileMonitor #Add a tag for nginx events - \$template LogglyFormatNginx,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 tag=\\\"nginx\\\"] %msg%\n\" + \$template LogglyFormatNginx,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 $TAG] %msg%\n\" if \$programname == 'nginx-access' then @@logs-01.loggly.com:514;LogglyFormatNginx if \$programname == 'nginx-access' then ~ @@ -296,7 +315,13 @@ checkIfNginxLogsMadeToLoggly() checkIfLogsAreParsedInLoggly() { nginxInitialLogCount=0 - queryParam="tag%3Anginx%20logtype%3Anginx&from=-15m&until=now&size=1" + TAG_PARSER= + IFS=, read -a array <<< "$LOGGLY_FILE_TAG" + for i in "${array[@]}" + do + TAG_PARSER="$TAG_PARSER%20tag%3A$i " + done + queryParam="logtype%3Anginx$TAG_PARSER&from=-15m&until=now&size=1" queryUrl="$LOGGLY_ACCOUNT_URL/apiv2/search?q=$queryParam" searchAndFetch nginxInitialLogCount "$queryUrl" logMsgToConfigSysLog "INFO" "INFO: Verifying if the Nginx logs are parsed in Loggly." @@ -322,7 +347,7 @@ remove21NginxConfFile() usage() { cat << EOF -usage: configure-nginx [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-s suppress prompts {optional)] +usage: configure-nginx [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-tag filetag1,filetag2 (optional)] [-s suppress prompts {optional)] usage: configure-nginx [-a loggly auth account or subdomain] [-r to rollback] usage: configure-nginx [-h for help] EOF @@ -351,6 +376,10 @@ while [ "$1" != "" ]; do -p | --password ) shift LOGGLY_PASSWORD=$1 ;; + -tag| --filetag ) shift + LOGGLY_FILE_TAG=$1 + echo "File tag: $LOGGLY_FILE_TAG" + ;; -r | --rollback ) LOGGLY_ROLLBACK="true" ;; From e083b056e921952508218572f3939b96c9c09bcb Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Thu, 27 Nov 2014 00:38:48 +0530 Subject: [PATCH 26/53] Added support to multiple tags --- Modular Scripts/Tomcat/configure-tomcat.sh | 42 +++++++++++++++++----- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/Modular Scripts/Tomcat/configure-tomcat.sh b/Modular Scripts/Tomcat/configure-tomcat.sh index 753c905..8fdcc5e 100644 --- a/Modular Scripts/Tomcat/configure-tomcat.sh +++ b/Modular Scripts/Tomcat/configure-tomcat.sh @@ -9,7 +9,7 @@ source configure-linux.sh "being-invoked" #name of the current script SCRIPT_NAME=configure-tomcat.sh #version of the current script -SCRIPT_VERSION=1.2 +SCRIPT_VERSION=1.3 #minimum version of tomcat to enable log rotation MIN_TOMCAT_VERSION=6.0.33.0 @@ -43,6 +43,12 @@ TOMCAT_VERSION= #this variable will hold the location of log4j files path LOG4J_FILE_PATH= +#tomcat as tag sent with the logs +LOGGLY_FILE_TAG="tomcat" + +#add tags to the logs +TAG= + #this variable will hold the catalina home provide by user. #this is not a mandatory input LOGGLY_CATALINA_HOME= @@ -91,7 +97,10 @@ installLogglyConfForTomcat() #update logging.properties file for log rotation updateLoggingPropertiesFile - + + #multiple tags + addTagsInConfiguration + #create 21tomcat.conf file write21TomcatConfFile @@ -418,6 +427,17 @@ EOIPFW fi } + +addTagsInConfiguration() +{ + #split tags by comman(,) + IFS=, read -a array <<< "$LOGGLY_FILE_TAG" + for i in "${array[@]}" + do + TAG="$TAG tag=\\\"$i\\\" " + done +} + write21TomcatConfFile() { #Create tomcat syslog config file if it doesn't exist @@ -465,7 +485,7 @@ write21TomcatFileContents() imfileStr+=" #parameterized token here....... #Add a tag for tomcat events - \$template LogglyFormatTomcat,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 tag=\\\"tomcat\\\"] %msg%\n\" + \$template LogglyFormatTomcat,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 $TAG] %msg%\n\" # catalina.out \$InputFileName $LOGGLY_CATALINA_LOG_HOME/catalina.out @@ -653,7 +673,7 @@ restartTomcat() usage() { cat << EOF -usage: configure-tomcat [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-ch catalina home (optional)] [-s suppress prompts {optional)] +usage: configure-tomcat [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-ch catalina home (optional)] [-tag filetag1,filetag2 (optional)] [-s suppress prompts {optional)] usage: configure-tomcat [-r to rollback] [-a loggly auth account or subdomain] [-ch catalina home (optional)] usage: configure-tomcat [-h for help] EOF @@ -685,13 +705,17 @@ while [ "$1" != "" ]; do ;; -p | --password ) shift LOGGLY_PASSWORD=$1 - ;; + ;; + -tag| --filetag ) shift + LOGGLY_FILE_TAG=$1 + echo "File tag: $LOGGLY_FILE_TAG" + ;; -r | --rollback ) - LOGGLY_ROLLBACK="true" + LOGGLY_ROLLBACK="true" ;; - -s | --suppress ) - SUPPRESS_PROMPT="true" - ;; + -s | --suppress ) + SUPPRESS_PROMPT="true" + ;; -h | --help) usage exit From 5b59acb9b5cbd35e8099830a0a8832c53b03fa56 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Tue, 2 Dec 2014 23:20:36 +0530 Subject: [PATCH 27/53] Linux Script: Bug Fixes. --- Linux Script/configure-linux.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Linux Script/configure-linux.sh b/Linux Script/configure-linux.sh index 60e86fe..059be6e 100644 --- a/Linux Script/configure-linux.sh +++ b/Linux Script/configure-linux.sh @@ -566,9 +566,11 @@ checkIfLogsMadeToLoggly() done if [ "$finalCount" -eq 1 ]; then - logMsgToConfigSysLog "SUCCESS" "SUCCESS: Verification logs successfully transferred to Loggly! You are now sending Linux system logs to Loggly." if [ "$IS_INVOKED" = "" ]; then + logMsgToConfigSysLog "SUCCESS" "SUCCESS: Verification logs successfully transferred to Loggly! You are now sending Linux system logs to Loggly." exit 0 + else + logMsgToConfigSysLog "INFO" "SUCCESS: Verification logs successfully transferred to Loggly! You are now sending Linux system logs to Loggly." fi fi From 4669afc9e07bb6c02a156b043c7cc76287462c9d Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Tue, 2 Dec 2014 23:28:15 +0530 Subject: [PATCH 28/53] Removed Log Rotation Check --- Modular Scripts/Apache2/configure-apache.sh | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/Modular Scripts/Apache2/configure-apache.sh b/Modular Scripts/Apache2/configure-apache.sh index de6975a..ee5abc6 100644 --- a/Modular Scripts/Apache2/configure-apache.sh +++ b/Modular Scripts/Apache2/configure-apache.sh @@ -126,9 +126,6 @@ checkApacheDetails() #set all the required apache variables by this script setApacheVariables - - #to check logrotation - checkIfLogRotationEnabled } #Get the apache service name on various linux flavors @@ -156,15 +153,6 @@ setApacheVariables() LOGGLY_APACHE_LOG_HOME=/var/log/$SERVICE } - -#checks if log rotation is enabled on the selected file -checkIfLogRotationEnabled() -{ - if [[ $(grep -r "/var/log/$SERVICE/*." /etc/logrotate.d/$SERVICE) ]]; then - logMsgToConfigSysLog "WARN" "WARN: Log rotation is enabled on $LOGGLY_APACHE_LOG_HOME/$APACHE_ACCESS_LOG_FILE and $LOGGLY_APACHE_LOG_HOME/$APACHE_ERROR_LOG_FILE. Please follow instructions here to update logrotate https://www.loggly.com/docs/log-rotate" - fi -} - #gets the version of apache installed on the unix box getApacheVersion() { From 8e33628446c1937d5a2d2323f241147491f9011f Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Tue, 2 Dec 2014 23:31:40 +0530 Subject: [PATCH 29/53] Removed Log Rotation Check --- .../File Monitoring/configure-file-monitoring.sh | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/Modular Scripts/File Monitoring/configure-file-monitoring.sh b/Modular Scripts/File Monitoring/configure-file-monitoring.sh index 1f95e16..5841283 100644 --- a/Modular Scripts/File Monitoring/configure-file-monitoring.sh +++ b/Modular Scripts/File Monitoring/configure-file-monitoring.sh @@ -59,9 +59,6 @@ installLogglyConfForFile() #check if file to monitor exists checkIfFileExist - - #check if logrotation enabled - checkIfLogRotationEnabled #checks if the file has proper read permission checkFileReadPermission @@ -153,15 +150,6 @@ checkIfFileExist() fi } -#checks if log rotation is enabled on the selected file -checkIfLogRotationEnabled() -{ - FILENAME="${LOGGLY_FILE_TO_MONITOR%.*}" - if [[ $(grep -r "$FILENAME" /etc/logrotate.d/) ]]; then - logMsgToConfigSysLog "WARN" "WARN: Log rotation is enabled on $LOGGLY_FILE_TO_MONITOR. Please follow instructions here to update logrotate https://www.loggly.com/docs/log-rotate" - fi -} - #check if the file alias is already taken checkIfFileAliasExist() { From b907287539ab0e8e29a3f8fdadbefcc20e4d4a1d Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Tue, 2 Dec 2014 23:34:14 +0530 Subject: [PATCH 30/53] Removed Log Rotation Check --- Modular Scripts/Nginx/configure-nginx.sh | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/Modular Scripts/Nginx/configure-nginx.sh b/Modular Scripts/Nginx/configure-nginx.sh index 44ad6a2..a5396e1 100644 --- a/Modular Scripts/Nginx/configure-nginx.sh +++ b/Modular Scripts/Nginx/configure-nginx.sh @@ -108,13 +108,6 @@ removeLogglyConfForNginx() logMsgToConfigSysLog "INFO" "INFO: Rollback completed." } -#checks if log rotation is enabled on the selected file -checkIfLogRotationEnabled() -{ - if [[ $(grep -r "/var/log/$SERVICE/*." /etc/logrotate.d/$SERVICE) ]]; then - logMsgToConfigSysLog "WARN" "WARN: Log rotation is enabled on $LOGGLY_NGINX_LOG_HOME/$NGINX_ACCESS_LOG_FILE and $LOGGLY_NGINX_LOG_HOME/$NGINX_ERROR_LOG_FILE. Please follow instructions here to update logrotate https://www.loggly.com/docs/log-rotate" - fi -} #identify if nginx is installed on your system and is available as a service checkNginxDetails() @@ -130,9 +123,6 @@ checkNginxDetails() #set all the required nginx variables by this script setNginxVariables - - #to check logrotation - checkIfLogRotationEnabled } From 245c946db3dd9b075f608bd99da31f7c7032274f Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Thu, 4 Dec 2014 00:03:52 +0530 Subject: [PATCH 31/53] Bug Fixes --- Modular Scripts/Apache2/configure-apache.sh | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/Modular Scripts/Apache2/configure-apache.sh b/Modular Scripts/Apache2/configure-apache.sh index ee5abc6..6354159 100644 --- a/Modular Scripts/Apache2/configure-apache.sh +++ b/Modular Scripts/Apache2/configure-apache.sh @@ -303,8 +303,19 @@ checkIfApacheLogsMadeToLoggly() apacheInitialLogCount=0 apacheLatestLogCount=0 - queryParam="tag%3Aapache&from=-15m&until=now&size=1" - + + TAGS= + IFS=, read -a array <<< "$LOGGLY_FILE_TAG" + for i in "${array[@]}" + do + if [ "$TAGS" == "" ]; then + TAGS="tag%3A$i" + else + TAGS="$TAGS%20tag%3A$i" + fi + done + + queryParam="$TAGS&from=-15m&until=now&size=1" queryUrl="$LOGGLY_ACCOUNT_URL/apiv2/search?q=$queryParam" logMsgToConfigSysLog "INFO" "INFO: Search URL: $queryUrl" From 0f2b7486fb070b37554d59b392c60ee519e0e45f Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Thu, 4 Dec 2014 00:08:29 +0530 Subject: [PATCH 32/53] Bug Fixes --- Modular Scripts/Nginx/configure-nginx.sh | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/Modular Scripts/Nginx/configure-nginx.sh b/Modular Scripts/Nginx/configure-nginx.sh index a5396e1..0d28df3 100644 --- a/Modular Scripts/Nginx/configure-nginx.sh +++ b/Modular Scripts/Nginx/configure-nginx.sh @@ -267,8 +267,19 @@ checkIfNginxLogsMadeToLoggly() nginxInitialLogCount=0 nginxLatestLogCount=0 - queryParam="tag%3Anginx&from=-15m&until=now&size=1" - + + TAGS= + IFS=, read -a array <<< "$LOGGLY_FILE_TAG" + for i in "${array[@]}" + do + if [ "$TAGS" == "" ]; then + TAGS="tag%3A$i" + else + TAGS="$TAGS%20tag%3A$i" + fi + done + + queryParam="$TAGS&from=-15m&until=now&size=1" queryUrl="$LOGGLY_ACCOUNT_URL/apiv2/search?q=$queryParam" logMsgToConfigSysLog "INFO" "INFO: Search URL: $queryUrl" From 2d26c06102b9a98cd2057fd290bbc8243a503431 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Tue, 16 Dec 2014 00:01:44 +0530 Subject: [PATCH 33/53] Remove state files on same alias Removes previous configuration state files and setup rsyslog if user wants to setup configuration using an existing alias. --- .../configure-file-monitoring.sh | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/Modular Scripts/File Monitoring/configure-file-monitoring.sh b/Modular Scripts/File Monitoring/configure-file-monitoring.sh index 5841283..ba1d591 100644 --- a/Modular Scripts/File Monitoring/configure-file-monitoring.sh +++ b/Modular Scripts/File Monitoring/configure-file-monitoring.sh @@ -9,7 +9,7 @@ source configure-linux.sh "being-invoked" #name of the current script SCRIPT_NAME=configure-file-monitoring.sh #version of the current script -SCRIPT_VERSION=1.9 +SCRIPT_VERSION=1.10 #file to monitor (contains complete path and file name) provided by user LOGGLY_FILE_TO_MONITOR= @@ -150,6 +150,16 @@ checkIfFileExist() fi } +#deletes the state file for the current alias, if exists +deleteStateFile() +{ + sudo rm -f $FILE_SYSLOG_CONFFILE + restartRsyslog + sudo rm -f $RSYSLOG_DIR/stat-$LOGGLY_FILE_TO_MONITOR_ALIAS + restartRsyslog +} + + #check if the file alias is already taken checkIfFileAliasExist() { @@ -161,7 +171,8 @@ checkIfFileAliasExist() case $yn in [Yy]* ) logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $FILE_SYSLOG_CONFFILE to $FILE_SYSLOG_CONFFILE_BACKUP"; - sudo mv -f $FILE_SYSLOG_CONFFILE $FILE_SYSLOG_CONFFILE_BACKUP; + sudo mv -f $FILE_SYSLOG_CONFFILE $FILE_SYSLOG_CONFFILE_BACKUP + deleteStateFile break;; [Nn]* ) logMsgToConfigSysLog "INFO" "INFO: Not overwriting the existing configuration. Exiting" @@ -172,7 +183,8 @@ checkIfFileAliasExist() done else logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $FILE_SYSLOG_CONFFILE to $FILE_SYSLOG_CONFFILE_BACKUP"; - sudo mv -f $FILE_SYSLOG_CONFFILE $FILE_SYSLOG_CONFFILE_BACKUP; + sudo mv -f $FILE_SYSLOG_CONFFILE $FILE_SYSLOG_CONFFILE_BACKUP + deleteStateFile fi fi } From d642646c11e8709b6d7bcb750671e7ddcb672577 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Wed, 21 Jan 2015 17:19:08 +0530 Subject: [PATCH 34/53] Bug Fixes --- Linux Script/configure-linux.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Linux Script/configure-linux.sh b/Linux Script/configure-linux.sh index 059be6e..513cc5c 100644 --- a/Linux Script/configure-linux.sh +++ b/Linux Script/configure-linux.sh @@ -368,7 +368,7 @@ checkIfRsyslogConfiguredAsService() exit 1 fi - if [ $(ps -ef | grep -v grep | grep "$RSYSLOG_SERVICE" | wc -l) -eq 0 ]; then + if [ $(ps -A | grep "$RSYSLOG_SERVICE" | wc -l) -eq 0 ]; then logMsgToConfigSysLog "INFO" "INFO: $RSYSLOG_SERVICE is not running. Attempting to start service." sudo service $RSYSLOG_SERVICE start fi @@ -385,7 +385,7 @@ checkifSyslogNgConfiguredAsService() #check if multiple versions of rsyslog is configured checkIfMultipleRsyslogConfigured() { - if [ $(ps -ef | grep -v grep | grep "$RSYSLOG_SERVICE" | wc -l) -gt 1 ]; then + if [ $(ps -A | grep "$RSYSLOG_SERVICE" | wc -l) -gt 1 ]; then logMsgToConfigSysLog "ERROR" "ERROR: Multiple (more than 1) $RSYSLOG_SERVICE is running." exit 1 fi @@ -568,10 +568,10 @@ checkIfLogsMadeToLoggly() if [ "$finalCount" -eq 1 ]; then if [ "$IS_INVOKED" = "" ]; then logMsgToConfigSysLog "SUCCESS" "SUCCESS: Verification logs successfully transferred to Loggly! You are now sending Linux system logs to Loggly." - exit 0 + exit 0 else logMsgToConfigSysLog "INFO" "SUCCESS: Verification logs successfully transferred to Loggly! You are now sending Linux system logs to Loggly." - fi + fi fi } From c382fc16a1a2670ff71773e34fa3919e2c3625fe Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Sat, 24 Jan 2015 00:16:48 +0530 Subject: [PATCH 35/53] Added troubleshooting rsyslog link --- Linux Script/configure-linux.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Linux Script/configure-linux.sh b/Linux Script/configure-linux.sh index 513cc5c..c9487c7 100644 --- a/Linux Script/configure-linux.sh +++ b/Linux Script/configure-linux.sh @@ -78,7 +78,8 @@ LOGGLY_DISTRIBUTION_ID="41058" #Instruction link on how to configure loggly on linux manually. This will get overwritten by the child script which calls this #on how to configure the child application -MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure rsyslog on Linux are available at https://www.loggly.com/docs/rsyslog-manual-configuration/." +MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure rsyslog on Linux are available at https://www.loggly.com/docs/rsyslog-manual-configuration/. Rsyslog troubleshooting instructions are available at https://www.loggly.com/docs/troubleshooting-rsyslog/" + #this variable is set if the script is invoked via some other calling script IS_INVOKED= @@ -559,7 +560,6 @@ checkIfLogsMadeToLoggly() searchAndFetch finalCount "$queryUrl" let counter=$counter+1 if [ "$counter" -gt "$maxCounter" ]; then - MANUAL_CONFIG_INSTRUCTION=$MANUAL_CONFIG_INSTRUCTION" Rsyslog troubleshooting instructions are available at https://www.loggly.com/docs/troubleshooting-rsyslog/" logMsgToConfigSysLog "ERROR" "ERROR: Logs did not make to Loggly in time. Please check network and firewall settings and retry." exit 1 fi From 4a6fba19644b023bab5d316173a27cdc21886f2e Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Sat, 24 Jan 2015 00:19:29 +0530 Subject: [PATCH 36/53] Added troubleshooting rsyslog link --- Modular Scripts/Apache2/configure-apache.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modular Scripts/Apache2/configure-apache.sh b/Modular Scripts/Apache2/configure-apache.sh index 6354159..48f30c9 100644 --- a/Modular Scripts/Apache2/configure-apache.sh +++ b/Modular Scripts/Apache2/configure-apache.sh @@ -32,7 +32,7 @@ LOGGLY_APACHE_LOG_HOME= #this variable will hold the users apache version APACHE_VERSION= -MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure Apache2 is available at https://www.loggly.com/docs/sending-apache-logs/" +MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure Apache2 is available at https://www.loggly.com/docs/sending-apache-logs/. Rsyslog troubleshooting instructions are available at https://www.loggly.com/docs/troubleshooting-rsyslog/" #this variable will hold if the check env function for linux is invoked APACHE_ENV_VALIDATED="false" From 352212d86f591a36fb889e3b1168fe46729af10e Mon Sep 17 00:00:00 2001 From: Jayant Varshney Date: Sat, 24 Jan 2015 00:21:11 +0530 Subject: [PATCH 37/53] Revert "Added troubleshooting rsyslog link" This reverts commit 4a6fba19644b023bab5d316173a27cdc21886f2e. --- Modular Scripts/Apache2/configure-apache.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modular Scripts/Apache2/configure-apache.sh b/Modular Scripts/Apache2/configure-apache.sh index 48f30c9..6354159 100644 --- a/Modular Scripts/Apache2/configure-apache.sh +++ b/Modular Scripts/Apache2/configure-apache.sh @@ -32,7 +32,7 @@ LOGGLY_APACHE_LOG_HOME= #this variable will hold the users apache version APACHE_VERSION= -MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure Apache2 is available at https://www.loggly.com/docs/sending-apache-logs/. Rsyslog troubleshooting instructions are available at https://www.loggly.com/docs/troubleshooting-rsyslog/" +MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure Apache2 is available at https://www.loggly.com/docs/sending-apache-logs/" #this variable will hold if the check env function for linux is invoked APACHE_ENV_VALIDATED="false" From af0be39a437de06c64f074f297d6dd7160099de3 Mon Sep 17 00:00:00 2001 From: Jayant Varshney Date: Sat, 24 Jan 2015 00:21:35 +0530 Subject: [PATCH 38/53] Revert "Added troubleshooting rsyslog link" This reverts commit c382fc16a1a2670ff71773e34fa3919e2c3625fe. --- Linux Script/configure-linux.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Linux Script/configure-linux.sh b/Linux Script/configure-linux.sh index c9487c7..513cc5c 100644 --- a/Linux Script/configure-linux.sh +++ b/Linux Script/configure-linux.sh @@ -78,8 +78,7 @@ LOGGLY_DISTRIBUTION_ID="41058" #Instruction link on how to configure loggly on linux manually. This will get overwritten by the child script which calls this #on how to configure the child application -MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure rsyslog on Linux are available at https://www.loggly.com/docs/rsyslog-manual-configuration/. Rsyslog troubleshooting instructions are available at https://www.loggly.com/docs/troubleshooting-rsyslog/" - +MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure rsyslog on Linux are available at https://www.loggly.com/docs/rsyslog-manual-configuration/." #this variable is set if the script is invoked via some other calling script IS_INVOKED= @@ -560,6 +559,7 @@ checkIfLogsMadeToLoggly() searchAndFetch finalCount "$queryUrl" let counter=$counter+1 if [ "$counter" -gt "$maxCounter" ]; then + MANUAL_CONFIG_INSTRUCTION=$MANUAL_CONFIG_INSTRUCTION" Rsyslog troubleshooting instructions are available at https://www.loggly.com/docs/troubleshooting-rsyslog/" logMsgToConfigSysLog "ERROR" "ERROR: Logs did not make to Loggly in time. Please check network and firewall settings and retry." exit 1 fi From b67a2c8a2a4203583738c823330ef3af20b46c83 Mon Sep 17 00:00:00 2001 From: Jayant Varshney Date: Sat, 24 Jan 2015 00:26:52 +0530 Subject: [PATCH 39/53] Added rsyslog troubleshooting link --- Linux Script/configure-linux.sh | 3 +-- Modular Scripts/Apache2/configure-apache.sh | 2 +- Modular Scripts/File Monitoring/configure-file-monitoring.sh | 2 +- Modular Scripts/Nginx/configure-nginx.sh | 2 +- .../S3Logs Monitoring/configure-s3-file-monitoring.sh | 2 +- Modular Scripts/Tomcat/configure-tomcat.sh | 2 +- 6 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Linux Script/configure-linux.sh b/Linux Script/configure-linux.sh index 513cc5c..fbdcdfc 100644 --- a/Linux Script/configure-linux.sh +++ b/Linux Script/configure-linux.sh @@ -78,7 +78,7 @@ LOGGLY_DISTRIBUTION_ID="41058" #Instruction link on how to configure loggly on linux manually. This will get overwritten by the child script which calls this #on how to configure the child application -MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure rsyslog on Linux are available at https://www.loggly.com/docs/rsyslog-manual-configuration/." +MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure rsyslog on Linux are available at https://www.loggly.com/docs/rsyslog-manual-configuration/. Rsyslog troubleshooting instructions are available at https://www.loggly.com/docs/troubleshooting-rsyslog/" #this variable is set if the script is invoked via some other calling script IS_INVOKED= @@ -559,7 +559,6 @@ checkIfLogsMadeToLoggly() searchAndFetch finalCount "$queryUrl" let counter=$counter+1 if [ "$counter" -gt "$maxCounter" ]; then - MANUAL_CONFIG_INSTRUCTION=$MANUAL_CONFIG_INSTRUCTION" Rsyslog troubleshooting instructions are available at https://www.loggly.com/docs/troubleshooting-rsyslog/" logMsgToConfigSysLog "ERROR" "ERROR: Logs did not make to Loggly in time. Please check network and firewall settings and retry." exit 1 fi diff --git a/Modular Scripts/Apache2/configure-apache.sh b/Modular Scripts/Apache2/configure-apache.sh index 6354159..48f30c9 100644 --- a/Modular Scripts/Apache2/configure-apache.sh +++ b/Modular Scripts/Apache2/configure-apache.sh @@ -32,7 +32,7 @@ LOGGLY_APACHE_LOG_HOME= #this variable will hold the users apache version APACHE_VERSION= -MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure Apache2 is available at https://www.loggly.com/docs/sending-apache-logs/" +MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure Apache2 is available at https://www.loggly.com/docs/sending-apache-logs/. Rsyslog troubleshooting instructions are available at https://www.loggly.com/docs/troubleshooting-rsyslog/" #this variable will hold if the check env function for linux is invoked APACHE_ENV_VALIDATED="false" diff --git a/Modular Scripts/File Monitoring/configure-file-monitoring.sh b/Modular Scripts/File Monitoring/configure-file-monitoring.sh index ba1d591..3a9e38e 100644 --- a/Modular Scripts/File Monitoring/configure-file-monitoring.sh +++ b/Modular Scripts/File Monitoring/configure-file-monitoring.sh @@ -26,7 +26,7 @@ FILE_SYSLOG_CONFFILE= #name and location of syslog backup file FILE_SYSLOG_CONFFILE_BACKUP= -MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure a file is available at https://www.loggly.com/docs/file-monitoring/" +MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure a file is available at https://www.loggly.com/docs/file-monitoring/. Rsyslog troubleshooting instructions are available at https://www.loggly.com/docs/troubleshooting-rsyslog/" #this variable is set if the script is invoked via some other calling script IS_FILE_MONITOR_SCRIPT_INVOKED="false" diff --git a/Modular Scripts/Nginx/configure-nginx.sh b/Modular Scripts/Nginx/configure-nginx.sh index 0d28df3..7ed1589 100644 --- a/Modular Scripts/Nginx/configure-nginx.sh +++ b/Modular Scripts/Nginx/configure-nginx.sh @@ -32,7 +32,7 @@ LOGGLY_NGINX_LOG_HOME= #this variable will hold the users nginx version NGINX_VERSION= -MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure nginx is available at https://www.loggly.com/docs/nginx-server-logs#manual" +MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure nginx is available at https://www.loggly.com/docs/nginx-server-logs#manual. Rsyslog troubleshooting instructions are available at https://www.loggly.com/docs/troubleshooting-rsyslog/" #this variable will hold if the check env function for linux is invoked NGINX_ENV_VALIDATED="false" diff --git a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh index 3a1bcde..f70108e 100644 --- a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh +++ b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh @@ -33,7 +33,7 @@ TEMP_DIR= IS_S3CMD_CONFIGURED_BY_SCRIPT="false" -MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure a file is available at https://www.loggly.com/docs/file-monitoring/" +MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure a file is available at https://www.loggly.com/docs/file-monitoring/. Rsyslog troubleshooting instructions are available at https://www.loggly.com/docs/troubleshooting-rsyslog/" ########## Variable Declarations - End ########## diff --git a/Modular Scripts/Tomcat/configure-tomcat.sh b/Modular Scripts/Tomcat/configure-tomcat.sh index 8fdcc5e..6a93d0d 100644 --- a/Modular Scripts/Tomcat/configure-tomcat.sh +++ b/Modular Scripts/Tomcat/configure-tomcat.sh @@ -53,7 +53,7 @@ TAG= #this is not a mandatory input LOGGLY_CATALINA_HOME= -MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure Tomcat is available at https://www.loggly.com/docs/tomcat-application-server" +MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure Tomcat is available at https://www.loggly.com/docs/tomcat-application-server/. Rsyslog troubleshooting instructions are available at https://www.loggly.com/docs/troubleshooting-rsyslog/" #this variable will hold if the check env function for linux is invoked TOMCAT_ENV_VALIDATED= From 95cb2ab9f962a0aba1e0bcef13fd47d30d8bd86a Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Wed, 4 Mar 2015 09:46:17 +0530 Subject: [PATCH 40/53] Added support to Disk Assisted queues --- Linux Script/configure-linux.sh | 129 +++++++++++++++----------------- 1 file changed, 62 insertions(+), 67 deletions(-) diff --git a/Linux Script/configure-linux.sh b/Linux Script/configure-linux.sh index fbdcdfc..7e2da65 100644 --- a/Linux Script/configure-linux.sh +++ b/Linux Script/configure-linux.sh @@ -15,7 +15,7 @@ function ctrl_c() { #name of the current script. This will get overwritten by the child script which calls this SCRIPT_NAME=configure-linux.sh #version of the current script. This will get overwritten by the child script which calls this -SCRIPT_VERSION=1.10 +SCRIPT_VERSION=1.12 #application tag. This will get overwritten by the child script which calls this APP_TAG= @@ -142,7 +142,7 @@ installLogglyConf() fi #if all the above check passes, write the 22-loggly.conf file - write22LogglyConfFile + checkAuthTokenAndWriteContents #create rsyslog dir if it doesn't exist, Modify the permission on rsyslog directory if exist on Ubuntu createRsyslogDir @@ -416,23 +416,10 @@ checkIfSelinuxServiceEnforced() fi } -#write 22-loggly,conf file to /etc/rsyslog.d directory after checking with user if override is needed -write22LogglyConfFile() -{ - echo "INFO: Checking if loggly sysconf file $LOGGLY_RSYSLOG_CONFFILE exist." - if [ -f "$LOGGLY_RSYSLOG_CONFFILE" ]; then - logMsgToConfigSysLog "INFO" "INFO: Loggly rsyslog file $LOGGLY_RSYSLOG_CONFFILE already exist." - checkIfConfigurationChanged - else - logMsgToConfigSysLog "INFO" "INFO: Loggly rsyslog file $LOGGLY_RSYSLOG_CONFFILE does not exist, creating file $LOGGLY_RSYSLOG_CONFFILE" - checkAuthTokenAndWriteContents - fi -} - #check if authentication token is valid and then write contents to 22-loggly.conf file to /etc/rsyslog.d directory checkAuthTokenAndWriteContents() { - if [ "$LOGGLY_ACCOUNT" != "" ]; then + if [ "$LOGGLY_AUTH_TOKEN" != "" ]; then writeContents $LOGGLY_ACCOUNT $LOGGLY_AUTH_TOKEN $LOGGLY_DISTRIBUTION_ID $LOGS_01_HOST $LOGGLY_SYSLOG_PORT restartRsyslog else @@ -441,57 +428,12 @@ checkAuthTokenAndWriteContents() fi } -#matches if the content of 22-loggly.conf content is changed -checkIfConfigurationChanged() -{ - ASK_FOR_VERIFICATION="false" - - #strings to be checked which should be present in the existing 22-loggly.conf. - #If these strings are not same then a warning message will be shown to user to update the 22-loggly.conf file - STR_TO_BE_CHECKED[0]="\$template LogglyFormat,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@$LOGGLY_DISTRIBUTION_ID] %msg%\n\"" - STR_TO_BE_CHECKED[1]="*.* @@$LOGS_01_HOST:$LOGGLY_SYSLOG_PORT;LogglyFormat" - - for i in "${STR_TO_BE_CHECKED[@]}" - do - if ! sudo grep -Fxq "$i" $LOGGLY_RSYSLOG_CONFFILE; then - ASK_FOR_VERIFICATION="true" - break; - fi - done - - if [ "$ASK_FOR_VERIFICATION" == "true" ]; then - logMsgToConfigSysLog "WARN" "WARN: Loggly rsyslog file /etc/rsyslog.d/22-loggly.conf content has changed." - if [ "$SUPPRESS_PROMPT" == "false" ]; then - while true; - do - read -p "Do you wish to override $LOGGLY_RSYSLOG_CONFFILE and re-verify configuration? (yes/no)" yn - case $yn in - [Yy]* ) - logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $LOGGLY_RSYSLOG_CONFFILE to $LOGGLY_RSYSLOG_CONFFILE_BACKUP"; - sudo mv -f $LOGGLY_RSYSLOG_CONFFILE $LOGGLY_RSYSLOG_CONFFILE_BACKUP; - checkAuthTokenAndWriteContents; - break;; - [Nn]* ) - LINUX_DO_VERIFICATION="false" - logMsgToConfigSysLog "INFO" "INFO: Skipping Linux verification." - break;; - * ) echo "Please answer yes or no.";; - esac - done - else - logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $LOGGLY_RSYSLOG_CONFFILE to $LOGGLY_RSYSLOG_CONFFILE_BACKUP"; - sudo mv -f $LOGGLY_RSYSLOG_CONFFILE $LOGGLY_RSYSLOG_CONFFILE_BACKUP; - checkAuthTokenAndWriteContents; - fi - else - LINUX_DO_VERIFICATION="false" - fi - -} #write the contents to 22-loggly.conf file writeContents() { + +WRITE_SCRIPT_CONTENTS="false" inputStr=" # ------------------------------------------------------- # Syslog Logging Directives for Loggly ($1.loggly.com) @@ -500,16 +442,67 @@ inputStr=" # Define the template used for sending logs to Loggly. Do not change this format. \$template LogglyFormat,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$2@$3] %msg%\n\" +\$WorkDirectory /var/spool/rsyslog # where to place spool files +\$ActionQueueFileName fwdRule1 # unique name prefix for spool files +\$ActionQueueMaxDiskSpace 1g # 1gb space limit (use as much as possible) +\$ActionQueueSaveOnShutdown on # save messages to disk on shutdown +\$ActionQueueType LinkedList # run asynchronously +\$ActionResumeRetryCount -1 # infinite retries if host is down + # Send messages to Loggly over TCP using the template. *.* @@$4:$5;LogglyFormat -# ------------------------------------------------------- -# End of Syslog Logging Directives for Loggly -# ------------------------------------------------------- +# ------------------------------------------------------- " + if [ -f "$LOGGLY_RSYSLOG_CONFFILE" ]; then + logMsgToConfigSysLog "INFO" "INFO: Loggly rsyslog file $LOGGLY_RSYSLOG_CONFFILE already exist." + + STR_SIZE=${#inputStr} + SIZE_FILE=$(stat -c%s "$LOGGLY_RSYSLOG_CONFFILE") + + #actual file size and variable size with same contents always differ in size with one byte + STR_SIZE=$(( STR_SIZE + 1 )) + + if [ "$STR_SIZE" -ne "$SIZE_FILE" ]; then + + logMsgToConfigSysLog "WARN" "WARN: Loggly rsyslog file /etc/rsyslog.d/22-loggly.conf content has changed." + if [ "$SUPPRESS_PROMPT" == "false" ]; then + while true; + do + read -p "Do you wish to override $LOGGLY_RSYSLOG_CONFFILE and re-verify configuration? (yes/no)" yn + case $yn in + [Yy]* ) + logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $LOGGLY_RSYSLOG_CONFFILE to $LOGGLY_RSYSLOG_CONFFILE_BACKUP"; + sudo mv -f $LOGGLY_RSYSLOG_CONFFILE $LOGGLY_RSYSLOG_CONFFILE_BACKUP; + WRITE_SCRIPT_CONTENTS="true" + break;; + [Nn]* ) + LINUX_DO_VERIFICATION="false" + logMsgToConfigSysLog "INFO" "INFO: Skipping Linux verification." + break;; + * ) echo "Please answer yes or no.";; + esac + done + else + logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $LOGGLY_RSYSLOG_CONFFILE to $LOGGLY_RSYSLOG_CONFFILE_BACKUP"; + sudo mv -f $LOGGLY_RSYSLOG_CONFFILE $LOGGLY_RSYSLOG_CONFFILE_BACKUP; + WRITE_SCRIPT_CONTENTS="true" + fi + else + LINUX_DO_VERIFICATION="false" + fi + else + WRITE_SCRIPT_CONTENTS="true" + fi + + if [ "$WRITE_SCRIPT_CONTENTS" == "true" ]; then + sudo cat << EOIPFW >> $LOGGLY_RSYSLOG_CONFFILE $inputStr EOIPFW + + fi + } #create /var/spool/rsyslog directory if not already present. Modify the permission of this directory for Ubuntu @@ -774,4 +767,6 @@ else IS_INVOKED="true" fi -########## Get Inputs from User - End ########## +########## Get Inputs from User - End ########## ------------------------------------------------------- +# End of Syslog Logging Directives for Loggly +# From bf14a8e0026af2f7ef30bbfb900d611fad92cf2c Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Wed, 4 Mar 2015 09:48:22 +0530 Subject: [PATCH 41/53] Added support to Directory Monitoring --- .../configure-file-monitoring.sh | 265 +++++++++++++++--- 1 file changed, 226 insertions(+), 39 deletions(-) diff --git a/Modular Scripts/File Monitoring/configure-file-monitoring.sh b/Modular Scripts/File Monitoring/configure-file-monitoring.sh index 3a9e38e..b794025 100644 --- a/Modular Scripts/File Monitoring/configure-file-monitoring.sh +++ b/Modular Scripts/File Monitoring/configure-file-monitoring.sh @@ -9,13 +9,15 @@ source configure-linux.sh "being-invoked" #name of the current script SCRIPT_NAME=configure-file-monitoring.sh #version of the current script -SCRIPT_VERSION=1.10 +SCRIPT_VERSION=1.11 #file to monitor (contains complete path and file name) provided by user LOGGLY_FILE_TO_MONITOR= #alias name, will be used as tag & state file name etc. provided by user LOGGLY_FILE_TO_MONITOR_ALIAS= +FILE_ALIAS= +STATE_FILE_ALIAS= #file alias provided by the user APP_TAG="\"file-alias\":\"\"" @@ -40,6 +42,11 @@ CONF_FILE_FORMAT_NAME="LogglyFormatFile" #add tags to the logs TAG= +FILE_TO_MONITOR= + +IS_DIRECTORY= + +IS_WILDCARD= ########## Variable Declarations - End ########## # executing the script for loggly to install and configure syslog @@ -53,37 +60,46 @@ installLogglyConfForFile() #checks if the file name contain spaces, if yes, the exit checkIfFileLocationContainSpaces - - #construct variables using filename and filealias - constructFileVariables - - #check if file to monitor exists - checkIfFileExist - - #checks if the file has proper read permission - checkFileReadPermission #check if the alias is already taken checkIfFileAliasExist - - #configure loggly for Linux - installLogglyConf - #multiple tags - addTagsInConfiguration + if [ "$IS_DIRECTORY" == "true" ]; then + + configureDirectoryFileMonitoring + + else + + #check if file to monitor exists + checkIfFileExist + + #construct variables using filename and filealias + constructFileVariables + + #check for the log file size + checkLogFileSize $LOGGLY_FILE_TO_MONITOR + + #checks if the file has proper read permission + checkFileReadPermission + + #configure loggly for Linux + installLogglyConf + + #multiple tags + addTagsInConfiguration + + #create 21.conf file + write21ConfFileContents + + fi - #create 21.conf file - write21ConfFileContents - #restart rsyslog restartRsyslog - - #check for the log file size - checkLogFileSize $LOGGLY_FILE_TO_MONITOR #verify if the file logs made it to loggly checkIfFileLogsMadeToLoggly + if [ "$IS_FILE_MONITOR_SCRIPT_INVOKED" = "false" ]; then #log success message logMsgToConfigSysLog "SUCCESS" "SUCCESS: Successfully configured to send $LOGGLY_FILE_TO_MONITOR logs via Loggly." @@ -113,6 +129,8 @@ removeLogglyConfForFile() #restart rsyslog restartRsyslog + removeStatFile + #log success message logMsgToConfigSysLog "INFO" "INFO: Rollback completed." } @@ -130,15 +148,105 @@ checkIfFileLocationContainSpaces() constructFileVariables() { #conf file name - FILE_SYSLOG_CONFFILE="$RSYSLOG_ETCDIR_CONF/21-filemonitoring-$LOGGLY_FILE_TO_MONITOR_ALIAS.conf" + FILE_SYSLOG_CONFFILE="$RSYSLOG_ETCDIR_CONF/21-filemonitoring-$FILE_ALIAS.conf" #conf file backup name - FILE_SYSLOG_CONFFILE_BACKUP="$FILE_SYSLOG_CONFFILE.loggly.bk" + FILE_SYSLOG_CONFFILE_BACKUP="$FILE_ALIAS.loggly.bk" #application tag APP_TAG="\"file-alias\":\"$LOGGLY_FILE_TO_MONITOR_ALIAS\"" } +#configures the directory files for file monitoring +configureDirectoryFileMonitoring() +{ + addTagsInConfiguration + TOTAL_FILES_IN_DIR=$(ls -1 ${LOGGLY_FILE_TO_MONITOR} | wc -l) + logMsgToConfigSysLog "INFO" "INFO: There are $TOTAL_FILES_IN_DIR files in directory. Configuring each file for monitoring present in this directory." + if [ "$SUPPRESS_PROMPT" == "false" ]; then + while true; do + read -p "There are $TOTAL_FILES_IN_DIR files present in this directory. Would you like to configure all the files (yes/no)?" yn + case $yn in + [Yy]* ) + installLogglyConf + for file in $( ls ${LOGGLY_FILE_TO_MONITOR} ) + do + configureFilesPresentInDirectory $file $FILE_ALIAS + done + break;; + [Nn]* ) + exit 1 + break;; + * ) echo "Please answer yes or no.";; + esac + done + while true; do + read -p "Would you like install a Cron job to sync the files currently in your Directory every 5 minutes? (yes/no)" yn + case $yn in + [Yy]* ) + doCronInstallation + break;; + [Nn]* ) + logMsgToConfigSysLog "INFO" "INFO: Skipping Cron installation." + break;; + * ) echo "Please answer yes or no.";; + esac + done + else + installLogglyConf + for file in $( ls ${LOGGLY_FILE_TO_MONITOR} ) + do + configureFilesPresentInDirectory $file $FILE_ALIAS + if [[ ! -f "$HOME/.loggly/file-monitoring-cron-$FILE_ALIAS.sh" ]]; then + doCronInstallation + fi + done + fi +} + +#actually configures a file present in the directory for monitoring +configureFilesPresentInDirectory() +{ + if [ "$IS_WILDCARD" == "true" ]; then + FILE_TO_MONITOR=$1 + else + FILE_TO_MONITOR=$LOGGLY_FILE_TO_MONITOR/$1 + fi + fileNameWithExt=${1##*/} + uniqueFileName=$(echo "$fileNameWithExt" | tr . _) + var=$(file $FILE_TO_MONITOR) + + #it may be possible that the "text" may contain some uppercase letters like "Text" + var=$(echo $var | tr "[:upper:]" "[:lower:]") + + if [[ $var == *text* ]]; then + LOGGLY_FILE_TO_MONITOR_ALIAS=$uniqueFileName-$2 + + if [ -f ${FILE_TO_MONITOR} ]; then + constructFileVariables + checkFileReadPermission + checkLogFileSize $FILE_TO_MONITOR + STATE_FILE_ALIAS=$(echo -n "$uniqueFileName" | md5sum | tr -d ' ')$FILE_ALIAS + write21ConfFileContents + fi + fi +} + +checkIfWildcardExist() +{ + TOTAL_FILES_IN_DIR=$(ls -1 ${LOGGLY_FILE_TO_MONITOR} 2> /dev/null | wc -l ) + WILDCARDS=( '*' '.' '?' '|' ']' '[' ) + for WILDCARD in "${WILDCARDS[@]}"; + do + if [[ $LOGGLY_FILE_TO_MONITOR == *"${WILDCARD}"* && $TOTAL_FILES_IN_DIR -gt 0 ]]; then + IS_WILDCARD="true" + return 0 + else + return 1 + fi + done +} + #checks if the file to be monitored exist checkIfFileExist() { @@ -153,13 +261,11 @@ checkIfFileExist() #deletes the state file for the current alias, if exists deleteStateFile() { - sudo rm -f $FILE_SYSLOG_CONFFILE restartRsyslog - sudo rm -f $RSYSLOG_DIR/stat-$LOGGLY_FILE_TO_MONITOR_ALIAS + sudo rm -f $RSYSLOG_DIR/stat-$FILE_ALIAS restartRsyslog } - #check if the file alias is already taken checkIfFileAliasExist() { @@ -217,15 +323,14 @@ checkLogFileSize() logMsgToConfigSysLog "WARN" "WARN: There are no recent logs from $LOGGLY_FILE_TO_MONITOR so there won't be any data sent to Loggly. You can generate some logs by writing to this file." exit 1 else - logMsgToConfigSysLog "INFO" "INFO: File size of $LOGGLY_FILE_TO_MONITOR is $monitorFileSize bytes." + logMsgToConfigSysLog "INFO" "INFO: File size of $FILE_TO_MONITOR is $monitorFileSize bytes." fi } #checks the input file has proper read permissions checkFileReadPermission() -{ - +{ LINUX_DIST_IN_LOWER_CASE=$(echo $LINUX_DIST | tr "[:upper:]" "[:lower:]") #no need to check read permissions with RedHat and CentOS as they also work with ---------- (000)permissions case "$LINUX_DIST_IN_LOWER_CASE" in @@ -237,15 +342,13 @@ checkFileReadPermission() FILE_PERMISSIONS=$(ls -l $LOGGLY_FILE_TO_MONITOR) #checking if the file has read permission for others PERMISSION_READ_OTHERS=${FILE_PERMISSIONS:7:1} - if [ $PERMISSION_READ_OTHERS != r ]; then + if [[ $PERMISSION_READ_OTHERS != r ]]; then logMsgToConfigSysLog "WARN" "WARN: $LOGGLY_FILE_TO_MONITOR does not have proper read permissions. Verification step may fail." fi ;; esac - } - addTagsInConfiguration() { #split tags by comman(,) @@ -256,6 +359,48 @@ addTagsInConfiguration() done } +doCronInstallation() +{ + if [[ ! -d "$HOME/.loggly" ]]; then + mkdir $HOME/.loggly + fi + CRON_SCRIPT="$HOME/.loggly/file-monitoring-cron-$FILE_ALIAS.sh" + logMsgToConfigSysLog "INFO" "INFO: Creating cron script $CRON_SCRIPT" + +sudo touch $CRON_SCRIPT +sudo chmod +x $CRON_SCRIPT + +cronScriptStr="#!/bin/bash +#curl -s -o configure-file-monitoring.sh https://www.loggly.com/install/configure-file-monitoring.sh + +sudo mv -f $FILE_SYSLOG_CONFFILE $FILE_SYSLOG_CONFFILE.bk +sudo rm -f $FILE_SYSLOG_CONFFILE + +sudo bash configure-file-monitoring.sh -a $LOGGLY_ACCOUNT -u $LOGGLY_USERNAME -p $LOGGLY_PASSWORD -f $LOGGLY_FILE_TO_MONITOR -l $FILE_ALIAS -tag $LOGGLY_FILE_TAG -s +" +#write to cron script file + +sudo cat << EOIPFW >> $CRON_SCRIPT +$cronScriptStr +EOIPFW + + CRON_JOB_TO_MONITOR_FILES="*/5 * * * * sudo bash $CRON_SCRIPT" + CRON_FILE="/tmp/File_Monitor_Cron" + + EXISTING_CRONS=$(sudo crontab -l 2>&1) + case $EXISTING_CRONS in + no*) + ;; + *) + echo "$EXISTING_CRONS" >> $CRON_FILE + ;; + esac + + echo "$CRON_JOB_TO_MONITOR_FILES" >> $CRON_FILE + sudo crontab $CRON_FILE + sudo rm -fr $CRON_FILE +} + #function to write the contents of syslog config file write21ConfFileContents() { @@ -263,7 +408,8 @@ write21ConfFileContents() sudo touch $FILE_SYSLOG_CONFFILE sudo chmod o+w $FILE_SYSLOG_CONFFILE - imfileStr="\$ModLoad imfile + imfileStr=" + \$ModLoad imfile \$InputFilePollInterval 10 \$WorkDirectory $RSYSLOG_DIR " @@ -274,9 +420,9 @@ write21ConfFileContents() imfileStr+=" # File access file: - \$InputFileName $LOGGLY_FILE_TO_MONITOR + \$InputFileName $FILE_TO_MONITOR \$InputFileTag $LOGGLY_FILE_TO_MONITOR_ALIAS: - \$InputFileStateFile stat-$LOGGLY_FILE_TO_MONITOR_ALIAS + \$InputFileStateFile stat-$STATE_FILE_ALIAS \$InputFileSeverity info \$InputFilePersistStateInterval 20000 \$InputRunFileMonitor @@ -336,6 +482,7 @@ checkIfFileLogsMadeToLoggly() checkIfLogsAreParsedInLoggly fi } + #verifying if the logs are being parsed or not checkIfLogsAreParsedInLoggly() { @@ -356,10 +503,11 @@ checkIfLogsAreParsedInLoggly() logMsgToConfigSysLog "WARN" "WARN: We received your logs but they do not appear to use one of our automatically parsed formats. You can still do full text search and counts on these logs, but you won't be able to use our field explorer. Please consider switching to one of our automated formats https://www.loggly.com/docs/automated-parsing/" fi } + #checks if the conf file exist. Name of conf file is constructed using the file alias name provided checkIfConfFileExist() { - if [ ! -f "$FILE_SYSLOG_CONFFILE" ]; then + if [[ ! -f "$FILE_SYSLOG_CONFFILE" ]]; then logMsgToConfigSysLog "ERROR" "ERROR: Invalid File Alias provided." exit 1 fi @@ -371,6 +519,7 @@ remove21ConfFile() echo "INFO: Deleting the loggly syslog conf file $FILE_SYSLOG_CONFFILE." if [ -f "$FILE_SYSLOG_CONFFILE" ]; then sudo rm -rf "$FILE_SYSLOG_CONFFILE" + deleteFileFromCrontab if [ "$IS_FILE_MONITOR_SCRIPT_INVOKED" = "false" ]; then echo "INFO: Removed all the modified files." fi @@ -379,6 +528,27 @@ remove21ConfFile() fi } +deleteFileFromCrontab() +{ + if [ -f "$HOME/.loggly/file-monitoring-cron-$FILE_ALIAS.sh" ];then + + logMsgToConfigSysLog "INFO" "INFO: Deleting sync Cron." + + #delete cron + sudo crontab -l | grep -v "$FILE_ALIAS" | crontab - + + #delete cron script + sudo rm -f $HOME/.loggly/file-monitoring-cron-$FILE_ALIAS.sh + + fi + +} + +removeStatFile() +{ + sudo rm -f $RSYSLOG_DIR/stat-*$FILE_ALIAS +} + #display usage syntax usage() { @@ -416,12 +586,29 @@ if [ "$1" != "being-invoked" ]; then LOGGLY_ROLLBACK="true" ;; -f | --filename ) shift - #LOGGLY_FILE_TO_MONITOR=$1 - LOGGLY_FILE_TO_MONITOR=$(readlink -f "$1") - echo "File to monitor: $LOGGLY_FILE_TO_MONITOR" + + LOGGLY_FILE_TO_MONITOR="${1%/}" + + if [ -f "$LOGGLY_FILE_TO_MONITOR" ];then + + LOGGLY_FILE_TO_MONITOR=$(readlink -f "$1") + FILE_TO_MONITOR=$LOGGLY_FILE_TO_MONITOR + echo "File to monitor: $LOGGLY_FILE_TO_MONITOR" + + elif [ -d "$LOGGLY_FILE_TO_MONITOR" ] || checkIfWildcardExist ; then + IS_DIRECTORY="true" + echo "Directory to monitor: $LOGGLY_FILE_TO_MONITOR" + + else + echo "ERROR: Cannot access $LOGGLY_FILE_TO_MONITOR: No such file or directory" + exit 1 + fi ;; -l | --filealias ) shift LOGGLY_FILE_TO_MONITOR_ALIAS=$1 + #keeping a copy of it as we need it in the loop + FILE_ALIAS=$LOGGLY_FILE_TO_MONITOR_ALIAS + STATE_FILE_ALIAS=$LOGGLY_FILE_TO_MONITOR_ALIAS CONF_FILE_FORMAT_NAME=$CONF_FILE_FORMAT_NAME$1 echo "File alias: $LOGGLY_FILE_TO_MONITOR_ALIAS" ;; From 884113ecaf19951279560c0a149e8676196207d1 Mon Sep 17 00:00:00 2001 From: Jayant Varshney Date: Wed, 4 Mar 2015 09:52:20 +0530 Subject: [PATCH 42/53] Added Mac Script --- Mac Script/configure-mac.sh | 677 ++++++++++++++++++++++++++++++++++++ 1 file changed, 677 insertions(+) create mode 100644 Mac Script/configure-mac.sh diff --git a/Mac Script/configure-mac.sh b/Mac Script/configure-mac.sh new file mode 100644 index 0000000..b06c1f5 --- /dev/null +++ b/Mac Script/configure-mac.sh @@ -0,0 +1,677 @@ +#!/bin/bash + +#trapping Control + C +#these statements must be the first statements in the script to trap the CTRL C event + +trap ctrl_c INT + +function ctrl_c() { + logMsgToConfigSysLog "INFO" "INFO: Aborting the script." + exit 1 +} + +########## Variable Declarations - Start ########## + +#name of the current script. This will get overwritten by the child script which calls this +SCRIPT_NAME=configure-mac.sh +#version of the current script. This will get overwritten by the child script which calls this +SCRIPT_VERSION=1.0 + +#application tag. This will get overwritten by the child script which calls this +APP_TAG= + +#this variable will hold the name of mac distribution +MAC_DIST= + +FLUENTD_CONF=loggly-fluentd.conf + +#ruby +RUBY=ruby + +#minimum version of ruby +MIN_RUBY_VERSION=1.9.3 + +#this variable will hold the users ruby version +RUBY_VERSION= + +#this variable will hold the host name +HOST_NAME= + +#host name for logs-01.loggly.com +LOGS_01_HOST=logs-01.loggly.com +LOGS_01_URL=https://$LOGS_01_HOST + +#this variable will contain loggly account url in the format https://$LOGGLY_ACCOUNT.loggly.com +LOGGLY_ACCOUNT_URL= + +#loggly.com URL +LOGGLY_COM_URL=https://www.loggly.com + +######Inputs provided by user###### +#this variable will hold the loggly account name provided by user. +#this is a mandatory input +LOGGLY_ACCOUNT= + +#this variable will hold the loggly authentication token provided by user. +#this is a mandatory input +LOGGLY_AUTH_TOKEN= + +#this variable will identify if the user has selected to rollback settings +LOGGLY_ROLLBACK= + +#this variable will hold the user name provided by user +#this is a mandatory input +LOGGLY_USERNAME= + +#this variable will hold the password provided by user +#this is a mandatory input +LOGGLY_PASSWORD= + +#variables used in fluentd.conf file +LOGGLY_SYSLOG_PORT=514 +LOGGLY_DISTRIBUTION_ID="41058" + +#this variable is set if the script is invoked via some other calling script +IS_INVOKED= + +#this variable will hold if the check env function for linux is invoked +MAC_ENV_VALIDATED="false" + +#this variable will inform if verification needs to be performed +MAC_DO_VERIFICATION="true" + +#if this variable is set to true then suppress all prompts +SUPPRESS_PROMPT="false" + +#plist file path +PROP_FILE= + +#manual instructions to be show in case of error +MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure rsyslog on Linux are available at https://www.loggly.com/docs/send-mac-logs-to-loggly/." + +checkMacLogglyCompatibility() +{ + #check if the user has root permission to run this script + checkIfUserHasRootPrivileges + + #check if the OS is supported by the script. If no, then exit + checkIfSupportedOS + + #set the basic variables needed by this script + setMacVariables + + #check if the Loggly servers are accessible. If no, ask user to check network connectivity & exit + checkIfLogglyServersAccessible + + #check if user credentials are valid. If no, then exit + checkIfValidUserNamePassword + + #get authentication token if not provided + getAuthToken + + #check if authentication token is valid. If no, then exit. + checkIfValidAuthToken + + #check if minimum version of ruby is installed + checkIfMinRubyVersionInstalled + + MAC_ENV_VALIDATED="true" +} + +# executing the script for loggly to install and configure fluentd. +installLogglyConf() +{ + #log message indicating starting of Loggly configuration + logMsgToConfigSysLog "INFO" "INFO: Initiating Configure Loggly for Mac." + + if [ "$MAC_ENV_VALIDATED" = "false" ]; then + checkMacLogglyCompatibility + fi + + #checks if fluentd already installed otherwise installs it + checkIfFluentdInstalled + + #installs Loggly fluentd plugin + installLogglyFluentdPlugin + + #write conf file in the system + writeLogglyConfFile + + #configures Fluentd service + configureFluentdAsService + + #starts Fluentd service + startFluentdService + + if [ "$MAC_DO_VERIFICATION" = "true" ]; then + #check if the logs are going to loggly fro mac system now + checkIfLogsMadeToLoggly + fi + + if [ "$IS_INVOKED" = "" ]; then + logMsgToConfigSysLog "SUCCESS" "SUCCESS: Mac system successfully configured to send logs via Loggly." + fi +} + + +#remove loggly configuration from Linux system +removeLogglyConf() +{ + #log message indicating starting of Loggly configuration + logMsgToConfigSysLog "INFO" "INFO: Initiating uninstall Loggly for Mac." + + #check if the user has root permission to run this script + checkIfUserHasRootPrivileges + + #check if the OS is supported by the script. If no, then exit + checkIfSupportedOS + + #set the basic variables needed by this script + setMacVariables + + #remove loggly.conf file + removeLogglyConfFile + + + #log success message + logMsgToConfigSysLog "SUCCESS" "SUCCESS: Uninstalled Loggly configuration from Mac system." +} + +#checks if the script is run as a sudo user +checkIfUserHasRootPrivileges() +{ + #This script needs to be run as a sudo user + if [[ $EUID -ne 0 ]]; then + logMsgToConfigSysLog "ERROR" "ERROR: This script must be run as root." + exit 1 + fi +} + + +#check if supported operating system +checkIfSupportedOS() +{ + # Determine OS platform + UNAME=$(uname | tr "[:upper:]" "[:lower:]") + MAC_DIST=$UNAME + if [ "$MAC_DIST" == "darwin" ]; then + logMsgToConfigSysLog "INFO" "INFO: Operating system is Mac" + else + logMsgToConfigSysLog "ERROR": "ERROR: This script supports only Mac systems. You can find alternative options here: https://www.loggly.com/docs/sending-logs-unixlinux-system-setup/" + exit 1 + fi +} + + +#sets mac variables which will be used across various functions +setMacVariables() +{ + #set host name + HOST_NAME=$(hostname) + + #set loggly account url + LOGGLY_ACCOUNT_URL=https://$LOGGLY_ACCOUNT.loggly.com +} + + +#checks if all the various endpoints used for configuring loggly are accessible +checkIfLogglyServersAccessible() +{ + logMsgToConfigSysLog "INFO" "INFO: Checking if $LOGS_01_HOST is reachable." + if [ $(ping -c 1 $LOGS_01_HOST | grep "1 packets transmitted, 1 packets received, 0.0% packet loss" | wc -l) == 1 ]; then + logMsgToConfigSysLog "INFO" "INFO: $LOGS_01_HOST is reachable." + else + logMsgToConfigSysLog "ERROR" "ERROR: $LOGS_01_HOST is not reachable. Please check your network and firewall settings." + exit 1 + fi + + echo "INFO: Checking if $LOGS_01_HOST is reachable via $LOGGLY_SYSLOG_PORT port. This may take some time." + if [ $(curl --connect-timeout 10 $LOGS_01_HOST:$LOGGLY_SYSLOG_PORT 2>&1 | grep "Empty reply from server" | wc -l) == 1 ]; then + echo "INFO: $LOGS_01_HOST is reachable via $LOGGLY_SYSLOG_PORT port." + else + logMsgToConfigSysLog "ERROR" "ERROR: $LOGS_01_HOST is not reachable via $LOGGLY_SYSLOG_PORT port. Please check your network and firewall settings." + exit 1 + fi + + echo "INFO: Checking if '$LOGGLY_ACCOUNT' subdomain is valid." + if [ $(curl -s --head --request GET $LOGGLY_ACCOUNT_URL/login | grep "200 OK" | wc -l) == 1 ]; then + echo "INFO: $LOGGLY_ACCOUNT_URL is valid and reachable." + else + logMsgToConfigSysLog "ERROR" "ERROR: This is not a recognized subdomain. Please ask the account owner for the subdomain they signed up with." + exit 1 + fi +} + +#check if user name and password is valid +checkIfValidUserNamePassword() +{ + echo "INFO: Checking if provided username and password is correct." + if [ $(curl -s -u $LOGGLY_USERNAME:$LOGGLY_PASSWORD $LOGGLY_ACCOUNT_URL/apiv2/customer | grep "Unauthorized" | wc -l) == 1 ]; then + logMsgToConfigSysLog "INFO" "INFO: Please check your username or reset your password at $LOGGLY_ACCOUNT_URL/account/users/" + logMsgToConfigSysLog "ERROR" "ERROR: Invalid Loggly username or password." + exit 1 + else + logMsgToConfigSysLog "INFO" "INFO: Username and password authorized successfully." + fi +} + +#gets the authentication token from the Loggly server +getAuthToken() +{ + if [ "$LOGGLY_AUTH_TOKEN" = "" ]; then + logMsgToConfigSysLog "INFO" "INFO: Authentication token not provided. Trying to retrieve it from $LOGGLY_ACCOUNT_URL account." + #get authentication token if user has not provided one + tokenstr=$(curl -s -u $LOGGLY_USERNAME:$LOGGLY_PASSWORD $LOGGLY_ACCOUNT_URL/apiv2/customer | grep -v "token") + + #get the string from index 0 to first occurence of , + tokenstr=${tokenstr%%,*} + + #get the string from index 0 to last occurence of " + tokenstr=${tokenstr%\"*} + + #get the string from first occurence of " to the end + tokenstr=${tokenstr#*\"} + + LOGGLY_AUTH_TOKEN=$tokenstr + + logMsgToConfigSysLog "INFO" "INFO: Retrieved authentication token: $LOGGLY_AUTH_TOKEN" + fi +} + + +#check if authentication token is valid +checkIfValidAuthToken() +{ + echo "INFO: Checking if provided auth token is correct." + if [ $(curl -s -u $LOGGLY_USERNAME:$LOGGLY_PASSWORD $LOGGLY_ACCOUNT_URL/apiv2/customer | grep \"$LOGGLY_AUTH_TOKEN\" | wc -l) == 1 ]; then + logMsgToConfigSysLog "INFO" "INFO: Authentication token validated successfully." + else + logMsgToConfigSysLog "ERROR" "ERROR: Invalid authentication token $LOGGLY_AUTH_TOKEN. You can get valid authentication token by following instructions at https://www.loggly.com/docs/customer-token-authentication-token/." + exit 1 + fi +} + +#this functions check if the min required version is installed in the system +checkIfMinRubyVersionInstalled() +{ + RUBY_VERSION=$(sudo $RUBY --version | grep "$RUBY") + RUBY_VERSION=${RUBY_VERSION%p*} + RUBY_VERSION=${RUBY_VERSION#* } + RUBY_VERSION=$RUBY_VERSION | tr -d " " + if [ $(compareVersions $RUBY_VERSION $MIN_RUBY_VERSION 3) -lt 0 ]; then + logMsgToConfigSysLog "ERROR" "ERROR: Min ruby version required is 1.9.3." + exit 1 + fi +} + +#this functions checks if the Fluentd gem is installed in the system +checkIfFluentdInstalled() +{ + if [ $(fluentd --setup ./fluent 2>/dev/null | grep "./fluent/fluent.conf" | wc -l ) == 1 ]; then + logMsgToConfigSysLog "INFO" "INFO: Fluentd is already installed. Not installing." + else + logMsgToConfigSysLog "INFO" "INFO: Fluentd is not installed. Installing Fluentd. This may take a while." + installFluentd + fi +} + +#this function installs the Fluentd in the system +installFluentd() +{ + #install fluentd gem http://docs.fluentd.org/articles/install-by-gem + sudo gem install fluentd --no-ri --no-rdoc + + #to check fluentd installed successfully + if [ $(fluentd --setup ./fluent 2>/dev/null | grep "./fluent/fluent.conf" | wc -l ) == 1 ]; then + logMsgToConfigSysLog "INFO" "INFO: Fluentd installed Successfully" + else + logMsgToConfigSysLog "ERROR" "ERROR: Unable to install fluentd" + exit 1 + fi +} + +#this function installs Loggly fluentd plugin +installLogglyFluentdPlugin() +{ + logMsgToConfigSysLog "INFO" "INFO: Installing Loggly plugin for Fluentd" + sudo gem install fluent-plugin-loggly + logMsgToConfigSysLog "INFO" "INFO: Loggly fluentd plugin installed successfully." +} + +#function to write the contents of fluentd config file +writeLogglyConfFile() +{ + + if [[ ! -d "$HOME/.loggly" ]]; then + mkdir $HOME/.loggly + fi + + FLUENTD_CONF="$HOME/.loggly/fluentd-loggly.conf" + + if [ -f "$FLUENTD_CONF" ]; then + echo "INFO: Conf file already exists. Creating Backup $FLUENTD_CONF $FLUENTD_CONF.bk" + sudo mv $FLUENTD_CONF $FLUENTD_CONF.bk + fi + + logMsgToConfigSysLog "INFO" "INFO: Creating file $FLUENTD_CONF" + + sudo touch $FLUENTD_CONF + +inputStr=" + + type tail + format none + path /var/log/system.log + tag system_logs + + + type loggly + loggly_url http://logs-01.loggly.com/inputs/$LOGGLY_AUTH_TOKEN/tag/Mac +" + +sudo cat << EOIPFW >> $FLUENTD_CONF +$inputStr +EOIPFW + +} + +#delete 22-loggly.conf file +removeLogglyConfFile() +{ + if [ -f "$HOME/.loggly/fluentd-loggly.conf" ]; then + logMsgToConfigSysLog "INFO" "INFO: Deleting file fluentd-loggly.conf" + sudo rm -rf "$HOME/.loggly/fluentd-loggly.conf" + + logMsgToConfigSysLog "INFO" "INFO: Removing Fluentd service" + sudo launchctl unload -F /Library/LaunchDaemons/com.loggly.loggly_fluentd.plist > /dev/null 2>&1 + sudo rm -rf /Library/LaunchDaemons/com.loggly.loggly_fluentd.plist + else + logMsgToConfigSysLog "ERROR" "ERROR: There is no conf file to delete" + exit 1 + fi +} + +#this function creates a fluentd daemon to send logs to Loggly +configureFluentdAsService() +{ + logMsgToConfigSysLog "INFO" "INFO: Creating daemon for Loggly conf file." + + #this sets the fluentd installation location + FLUENTD_LOCATION=$(which fluentd) + + PROP_FILE="/Library/LaunchDaemons/com.loggly.loggly_fluentd.plist" + + #if loggly fluentd is already running as a service then unload it + if [ $(sudo launchctl list | grep 'com.loggly.loggly_fluentd' | wc -l) == 1 ]; then + sudo launchctl unload -F $PROP_FILE > /dev/null 2>&1 + + #if there was some error while unloading, just remove it + sudo launchctl remove com.loggly.loggly_fluentd > /dev/null 2>&1 + fi + + #if plist file is already there then delete it + if [ -f "$PROP_FILE" ]; then + sudo rm -f $PROP_FILE + fi + + sudo touch $PROP_FILE + sudo chmod +x $PROP_FILE + +propStr=" + + + + Label + com.loggly.loggly_fluentd + ProgramArguments + + $FLUENTD_LOCATION + -c + $HOME/.loggly/fluentd-loggly.conf + + RunAtLoad + + StandardErrorPath + /tmp/loggly_fluentd.err + StandardOutPath + /tmp/loggly_fluentd.out + +" + +sudo cat << EOIPFW >> $PROP_FILE +$propStr +EOIPFW + +} + +#starts Fluentd Service +startFluentdService() +{ + logMsgToConfigSysLog "INFO" "INFO: Starting Fluentd as a service" + sudo launchctl load -F $PROP_FILE + logMsgToConfigSysLog "INFO" "INFO: Fluentd started successfully" +} + +#check if the logs made it to Loggly +checkIfLogsMadeToLoggly() +{ + logMsgToConfigSysLog "INFO" "INFO: Sending test message to Loggly." + uuid=$(cat /dev/urandom | env LC_CTYPE=C tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1) + + queryParam="tag%3AMac%20$uuid" + logger -t "Mac" "Mac-Test message for verification with UUID $uuid" + + counter=1 + maxCounter=10 + finalCount=0 + + queryUrl="$LOGGLY_ACCOUNT_URL/apiv2/search?q=$queryParam" + logMsgToConfigSysLog "INFO" "INFO: Search URL: $queryUrl" + + logMsgToConfigSysLog "INFO" "INFO: Verifying if the log made it to Loggly." + logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." + searchAndFetch finalCount "$queryUrl" + let counter=$counter+1 + + while [ "$finalCount" -eq 0 ]; do + echo "INFO: Did not find the test log message in Loggly's search yet. Waiting for 30 secs." + sleep 30 + echo "INFO: Done waiting. Verifying again." + logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." + searchAndFetch finalCount "$queryUrl" + let counter=$counter+1 + if [ "$counter" -gt "$maxCounter" ]; then + logMsgToConfigSysLog "ERROR" "ERROR: Logs did not make to Loggly in time. Please check network and firewall settings and retry." + exit 1 + fi + done + + if [ "$finalCount" -eq 1 ]; then + if [ "$IS_INVOKED" = "" ]; then + logMsgToConfigSysLog "SUCCESS" "SUCCESS: Verification logs successfully transferred to Loggly! You are now sending Mac system logs to Loggly." + exit 0 + else + logMsgToConfigSysLog "INFO" "SUCCESS: Verification logs successfully transferred to Loggly! You are now sending Mac system logs to Loggly." + fi + fi + +} + +compareVersions () +{ + typeset IFS='.' + typeset -a v1=( $1 ) + typeset -a v2=( $2 ) + typeset n diff + + for (( n=0; n<$3; n+=1 )); do + diff=$((v1[n]-v2[n])) + if [ $diff -ne 0 ] ; then + [ $diff -le 0 ] && echo '-1' || echo '1' + return + fi + done + echo '0' +} + + +#logs message to config syslog +logMsgToConfigSysLog() +{ + #$1 variable will be SUCCESS or ERROR or INFO or WARNING + #$2 variable will be the message + cslStatus=$1 + cslMessage=$2 + echo "$cslMessage" + currentTime=$(date) + + #for Linux system, we need to use -d switch to decode base64 whereas + #for Mac system, we need to use -D switch to decode + varUname=$(uname) + if [[ $varUname == 'Linux' ]]; then + enabler=$(echo -n MWVjNGU4ZTEtZmJiMi00N2U3LTkyOWItNzVhMWJmZjVmZmUw | base64 -d) + elif [[ $varUname == 'Darwin' ]]; then + enabler=$(echo MWVjNGU4ZTEtZmJiMi00N2U3LTkyOWItNzVhMWJmZjVmZmUw | base64 -D) + fi + + if [ $? -ne 0 ]; then + echo "ERROR: Base64 decode is not supported on your Operating System. Please update your system to support Base64." + exit 1 + fi + + sendPayloadToConfigSysLog "$cslStatus" "$cslMessage" "$enabler" + + #if it is an error, then log message "Script Failed" to config syslog and exit the script + if [[ $cslStatus == "ERROR" ]]; then + sendPayloadToConfigSysLog "ERROR" "Script Failed" "$enabler" + echo $MANUAL_CONFIG_INSTRUCTION + exit 1 + fi + + #if it is a success, then log message "Script Succeeded" to config syslog and exit the script + if [[ $cslStatus == "SUCCESS" ]]; then + sendPayloadToConfigSysLog "SUCCESS" "Script Succeeded" "$enabler" + exit 0 + fi +} + + +#payload construction to send log to config syslog +sendPayloadToConfigSysLog() +{ + if [ "$APP_TAG" = "" ]; then + var="{\"sub-domain\":\"$LOGGLY_ACCOUNT\", \"user-name\":\"$LOGGLY_USERNAME\", \"customer-token\":\"$LOGGLY_AUTH_TOKEN\", \"host-name\":\"$HOST_NAME\", \"script-name\":\"$SCRIPT_NAME\", \"script-version\":\"$SCRIPT_VERSION\", \"status\":\"$1\", \"time-stamp\":\"$currentTime\", \"Mac-distribution\":\"$MAC_DIST\", \"messages\":\"$2\",\"ruby-version\":\"$RUBY_VERSION\"}" + else + var="{\"sub-domain\":\"$LOGGLY_ACCOUNT\", \"user-name\":\"$LOGGLY_USERNAME\", \"customer-token\":\"$LOGGLY_AUTH_TOKEN\", \"host-name\":\"$HOST_NAME\", \"script-name\":\"$SCRIPT_NAME\", \"script-version\":\"$SCRIPT_VERSION\", \"status\":\"$1\", \"time-stamp\":\"$currentTime\", \"Mac-distribution\":\"$MAC_DIST\", $APP_TAG, \"messages\":\"$2\",\"ruby-version\":\"$RUBY_VERSION\"}" + fi + curl -s -H "content-type:application/json" -d "$var" $LOGS_01_URL/inputs/$3 > /dev/null 2>&1 +} + +#$1 return the count of records in loggly, $2 is the query param to search in loggly +searchAndFetch() +{ + url=$2 + + result=$(curl -s -u "$LOGGLY_USERNAME":"$LOGGLY_PASSWORD" "$url" ) + + if [ -z "$result" ]; then + logMsgToConfigSysLog "ERROR" "ERROR: Please check your network/firewall settings & ensure Loggly subdomain, username and password is specified correctly." + exit 1 + fi + id=$(echo "$result" | grep -v "{" | grep id | awk '{print $2}') + # strip last double quote from id + id="${id%\"}" + # strip first double quote from id + id="${id#\"}" + url="$LOGGLY_ACCOUNT_URL/apiv2/events?rsid=$id" + + # retrieve the data + result=$(curl -s -u "$LOGGLY_USERNAME":"$LOGGLY_PASSWORD" "$url" ) + count=$(echo "$result" | grep total_events | awk '{print $2}') + count="${count%\,}" + eval $1="'$count'" + if [ "$count" -gt 0 ]; then + timestamp=$(echo "$result" | grep timestamp) + fi +} + +#get password in the form of asterisk +getPassword() +{ + unset LOGGLY_PASSWORD + prompt="Please enter Loggly Password:" + while IFS= read -p "$prompt" -r -s -n 1 char + do + if [[ $char == $'\0' ]] + then + break + fi + prompt='*' + LOGGLY_PASSWORD+="$char" + done + echo +} + +#display usage syntax +usage() +{ +cat << EOF +usage: configure-mac [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] +usage: configure-mac [-a loggly auth account or subdomain] [-r to remove] +usage: configure-mac [-h for help] +EOF +} + +########## Get Inputs from User - Start ########## +if [ "$1" != "being-invoked" ]; then + if [ $# -eq 0 ]; then + usage + exit + else + while [ "$1" != "" ]; do + case $1 in + -t | --token ) shift + LOGGLY_AUTH_TOKEN=$1 + echo "AUTH TOKEN $LOGGLY_AUTH_TOKEN" + ;; + -a | --account ) shift + LOGGLY_ACCOUNT=$1 + echo "Loggly account or subdomain: $LOGGLY_ACCOUNT" + ;; + -u | --username ) shift + LOGGLY_USERNAME=$1 + echo "Username is set" + ;; + -p | --password ) shift + LOGGLY_PASSWORD=$1 + ;; + -r | --remove ) + LOGGLY_REMOVE="true" + ;; + -h | --help) + usage + exit + ;; + *) usage + exit + ;; + esac + shift + done + fi + + if [ "$LOGGLY_REMOVE" != "" -a "$LOGGLY_ACCOUNT" != "" ]; then + removeLogglyConf + elif [ "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_USERNAME" != "" ]; then + if [ "$LOGGLY_PASSWORD" = "" ]; then + getPassword + fi + installLogglyConf + else + usage + fi +else + IS_INVOKED="true" +fi + +########## Get Inputs from User - End ########## From ded22fe9770c23e3e99d372233243563fb10738f Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Wed, 4 Mar 2015 09:54:56 +0530 Subject: [PATCH 43/53] Added README --- Mac Script/README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 Mac Script/README.md diff --git a/Mac Script/README.md b/Mac Script/README.md new file mode 100644 index 0000000..012519f --- /dev/null +++ b/Mac Script/README.md @@ -0,0 +1,10 @@ +Mac Script +============ + +Configure your Mac system to send syslogs to Loggly using the following command + + sudo bash configure-mac.sh -a SUBDOMAIN -u USERNAME + +Stop sending your Linux System logs to Loggly + + sudo bash configure-mac.sh -a SUBDOMAIN -r From 3215989b4c6853f99174d618bdd4569c6a55df48 Mon Sep 17 00:00:00 2001 From: Jayant Varshney Date: Wed, 18 Mar 2015 10:02:07 +0530 Subject: [PATCH 44/53] Bug Fixes 1. Updated File monitoring to support recursive directories 2. Made s3 script compatible to updated file monitoring script --- .../configure-file-monitoring.sh | 43 ++++++--------- .../configure-s3-file-monitoring.sh | 55 +++++++------------ 2 files changed, 38 insertions(+), 60 deletions(-) diff --git a/Modular Scripts/File Monitoring/configure-file-monitoring.sh b/Modular Scripts/File Monitoring/configure-file-monitoring.sh index b794025..04d0cd7 100644 --- a/Modular Scripts/File Monitoring/configure-file-monitoring.sh +++ b/Modular Scripts/File Monitoring/configure-file-monitoring.sh @@ -169,8 +169,8 @@ configureDirectoryFileMonitoring() case $yn in [Yy]* ) installLogglyConf - for file in $( ls ${LOGGLY_FILE_TO_MONITOR} ) - do + for file in $(find $LOGGLY_FILE_TO_MONITOR -name '*') + do configureFilesPresentInDirectory $file $FILE_ALIAS done break;; @@ -194,34 +194,29 @@ configureDirectoryFileMonitoring() done else installLogglyConf - for file in $( ls ${LOGGLY_FILE_TO_MONITOR} ) - do + for file in $(find $LOGGLY_FILE_TO_MONITOR -name '*') + do configureFilesPresentInDirectory $file $FILE_ALIAS - if [[ ! -f "$HOME/.loggly/file-monitoring-cron-$FILE_ALIAS.sh" ]]; then - doCronInstallation - fi - done + done + if [[ ! -f "/root/.loggly/file-monitoring-cron-$FILE_ALIAS.sh" ]]; then + doCronInstallation + fi fi } #actually configures a file present in the directory for monitoring configureFilesPresentInDirectory() { - if [ "$IS_WILDCARD" == "true" ]; then - FILE_TO_MONITOR=$1 - else - FILE_TO_MONITOR=$LOGGLY_FILE_TO_MONITOR/$1 - fi + FILE_TO_MONITOR=$1 fileNameWithExt=${1##*/} uniqueFileName=$(echo "$fileNameWithExt" | tr . _) var=$(file $FILE_TO_MONITOR) - + + #checking if it is a text file otherwise ignore it #it may be possible that the "text" may contain some uppercase letters like "Text" var=$(echo $var | tr "[:upper:]" "[:lower:]") - if [[ $var == *text* ]]; then LOGGLY_FILE_TO_MONITOR_ALIAS=$uniqueFileName-$2 - if [ -f ${FILE_TO_MONITOR} ]; then constructFileVariables checkFileReadPermission @@ -361,21 +356,19 @@ addTagsInConfiguration() doCronInstallation() { - if [[ ! -d "$HOME/.loggly" ]]; then - mkdir $HOME/.loggly + if [[ ! -d "/root/.loggly" ]]; then + mkdir /root/.loggly fi - CRON_SCRIPT="$HOME/.loggly/file-monitoring-cron-$FILE_ALIAS.sh" + CRON_SCRIPT="/root/.loggly/file-monitoring-cron-$FILE_ALIAS.sh" logMsgToConfigSysLog "INFO" "INFO: Creating cron script $CRON_SCRIPT" sudo touch $CRON_SCRIPT sudo chmod +x $CRON_SCRIPT cronScriptStr="#!/bin/bash -#curl -s -o configure-file-monitoring.sh https://www.loggly.com/install/configure-file-monitoring.sh - +curl -s -o configure-file-monitoring.sh https://www.loggly.com/install/configure-file-monitoring.sh sudo mv -f $FILE_SYSLOG_CONFFILE $FILE_SYSLOG_CONFFILE.bk sudo rm -f $FILE_SYSLOG_CONFFILE - sudo bash configure-file-monitoring.sh -a $LOGGLY_ACCOUNT -u $LOGGLY_USERNAME -p $LOGGLY_PASSWORD -f $LOGGLY_FILE_TO_MONITOR -l $FILE_ALIAS -tag $LOGGLY_FILE_TAG -s " #write to cron script file @@ -530,15 +523,15 @@ remove21ConfFile() deleteFileFromCrontab() { - if [ -f "$HOME/.loggly/file-monitoring-cron-$FILE_ALIAS.sh" ];then + if [ -f "/root/.loggly/file-monitoring-cron-$FILE_ALIAS.sh" ];then logMsgToConfigSysLog "INFO" "INFO: Deleting sync Cron." #delete cron - sudo crontab -l | grep -v "$FILE_ALIAS" | crontab - + sudo crontab -l | grep -v "file-monitoring-cron-$FILE_ALIAS.sh" | crontab - #delete cron script - sudo rm -f $HOME/.loggly/file-monitoring-cron-$FILE_ALIAS.sh + sudo rm -f /root/.loggly/file-monitoring-cron-$FILE_ALIAS.sh fi diff --git a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh index f70108e..c2c57f3 100644 --- a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh +++ b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh @@ -8,13 +8,21 @@ source configure-file-monitoring.sh "being-invoked" #name of the current script SCRIPT_NAME=configure-s3-file-monitoring.sh #version of the current script -SCRIPT_VERSION=1.4 +SCRIPT_VERSION=1.5 #s3 bucket name to configure LOGGLY_S3_BUCKET_NAME= #alias name, will be used as tag & state file name etc. provided by user LOGGLY_S3_ALIAS= +FILE_ALIAS= +STATE_FILE_ALIAS= + +#file as tag sent with the logs +LOGGLY_FILE_TAG="s3file" + +#format name for the conf file. Can be set by calling script +CONF_FILE_FORMAT_NAME="LogglyFormatS3" #file alias provided by the user APP_TAG="\"s3file-alias\":\"\"" @@ -55,9 +63,6 @@ installLogglyConfForS3() #check if s3bucket is valid checkIfValidS3Bucket - #configure loggly for Linux - installLogglyConf - #create temporary directory createTempDir @@ -250,37 +255,15 @@ downloadS3Bucket() fi } - invokeS3FileMonitoring() { dir=/tmp/s3monitoring/$LOGGLY_S3_ALIAS - #TODO: Not supporting multiple files with same name in different directories - #only supporting file with naming convention *.* - for f in $(find $dir -name '*') - do - fileNameWithExt=${f##*/} - uniqueFileName=$(echo "$fileNameWithExt" | tr . _) - var=$(file $f) - - #it may be possible that the "text" may contain some uppercase letters like "Text" - var=$(echo $var | tr "[:upper:]" "[:lower:]") - - if [[ $var == *text* ]]; then - LOGGLY_FILE_TO_MONITOR_ALIAS=$uniqueFileName-$LOGGLY_S3_ALIAS - LOGGLY_FILE_TO_MONITOR=$f - LOGGLY_FILE_TAG="s3file" - CONF_FILE_FORMAT_NAME="LogglyFormatS3" - constructFileVariables - checkFileReadPermission - checkLogFileSize $LOGGLY_FILE_TO_MONITOR - write21ConfFileContents - IS_ANY_FILE_CONFIGURED="true" - #ignoring directory - elif [[ $var != *directory* ]]; then - logMsgToConfigSysLog "WARN" "WARN: File $fileNameWithExt is not a text file. Ignoring." - fi - done + LOGGLY_FILE_TO_MONITOR=$dir + configureDirectoryFileMonitoring + + IS_ANY_FILE_CONFIGURED="true" + if [ "$IS_ANY_FILE_CONFIGURED" != "false" ]; then restartRsyslog fi @@ -293,7 +276,7 @@ installCronToSyncS3BucketPeriodically() read -p "Would you like install a Cron job to sync the files currently in your bucket every 5 minutes? (yes/no)" yn case $yn in [Yy]* ) - doCronInstallation + doS3CronInstallation break;; [Nn]* ) logMsgToConfigSysLog "INFO" "INFO: Skipping Cron installation." @@ -302,11 +285,11 @@ installCronToSyncS3BucketPeriodically() esac done else - doCronInstallation + doS3CronInstallation fi } -doCronInstallation() +doS3CronInstallation() { #copying .s3cfg file to /root so that it can be used by crontab for sync if ! sudo test -f "/root/.s3cfg" ; then @@ -477,7 +460,9 @@ while [ "$1" != "" ]; do -s3l | --s3alias ) shift LOGGLY_S3_ALIAS=$1 - echo "File alias: $LOGGLY_S3_ALIAS" + FILE_ALIAS=$LOGGLY_S3_ALIAS + STATE_FILE_ALIAS=$LOGGLY_S3_ALIAS + echo "File alias: $FILE_ALIAS" ;; -s | --suppress ) SUPPRESS_PROMPT="true" From 62fce9714e975ee2b380caaa7614ed50d68673c9 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Tue, 24 Mar 2015 16:58:27 +0530 Subject: [PATCH 45/53] Bug fixes for syslog-ng detection --- Linux Script/configure-linux.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Linux Script/configure-linux.sh b/Linux Script/configure-linux.sh index 7e2da65..89642ef 100644 --- a/Linux Script/configure-linux.sh +++ b/Linux Script/configure-linux.sh @@ -363,11 +363,13 @@ checkIfRsyslogConfiguredAsService() if [ -f /etc/init.d/$RSYSLOG_SERVICE ]; then logMsgToConfigSysLog "INFO" "INFO: $RSYSLOG_SERVICE is present as service." else - checkifSyslogNgConfiguredAsService logMsgToConfigSysLog "ERROR" "ERROR: $RSYSLOG_SERVICE is not present as service." exit 1 fi - + + #checking if syslog-ng is running as a service + checkifSyslogNgConfiguredAsService + if [ $(ps -A | grep "$RSYSLOG_SERVICE" | wc -l) -eq 0 ]; then logMsgToConfigSysLog "INFO" "INFO: $RSYSLOG_SERVICE is not running. Attempting to start service." sudo service $RSYSLOG_SERVICE start From 164b2f41780f2aa5b7c71cb5bffba340311b6fec Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Fri, 27 Mar 2015 16:11:13 +0530 Subject: [PATCH 46/53] Improved syslog-ng detection --- Linux Script/configure-linux.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Linux Script/configure-linux.sh b/Linux Script/configure-linux.sh index 89642ef..8c71bbb 100644 --- a/Linux Script/configure-linux.sh +++ b/Linux Script/configure-linux.sh @@ -116,6 +116,9 @@ checkLinuxLogglyCompatibility() #check if authentication token is valid. If no, then exit. checkIfValidAuthToken + #checking if syslog-ng is configured as a service + checkifSyslogNgConfiguredAsService + #check if rsyslog is configured as service. If no, then exit checkIfRsyslogConfiguredAsService @@ -378,7 +381,7 @@ checkIfRsyslogConfiguredAsService() checkifSyslogNgConfiguredAsService() { - if [ -f /etc/init.d/$SYSLOG_NG_SERVICE ]; then + if [ $(ps -A | grep "$SYSLOG_NG_SERVICE" | wc -l) -gt 0 ]; then logMsgToConfigSysLog "ERROR" "ERROR: This script does not currently support syslog-ng. Please follow the instructions on this page https://www.loggly.com/docs/syslog-ng-manual-configuration" exit 1 fi @@ -393,7 +396,7 @@ checkIfMultipleRsyslogConfigured() fi } -#check if mimimum version of rsyslog required to configure loggly is met +#check if minimum version of rsyslog required to configure loggly is met checkIfMinVersionOfRsyslog() { RSYSLOG_VERSION=$(sudo $RSYSLOGD -version | grep "$RSYSLOGD") From 2a44babd42087c680aa34dc745492add995beb6d Mon Sep 17 00:00:00 2001 From: Jayant Varshney Date: Fri, 27 Mar 2015 23:54:46 +0530 Subject: [PATCH 47/53] Improved error text --- Linux Script/configure-linux.sh | 2 +- Mac Script/configure-mac.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Linux Script/configure-linux.sh b/Linux Script/configure-linux.sh index 8c71bbb..13966fd 100644 --- a/Linux Script/configure-linux.sh +++ b/Linux Script/configure-linux.sh @@ -319,7 +319,7 @@ checkIfValidUserNamePassword() echo "INFO: Checking if provided username and password is correct." if [ $(curl -s -u $LOGGLY_USERNAME:$LOGGLY_PASSWORD $LOGGLY_ACCOUNT_URL/apiv2/customer | grep "Unauthorized" | wc -l) == 1 ]; then logMsgToConfigSysLog "INFO" "INFO: Please check your username or reset your password at $LOGGLY_ACCOUNT_URL/account/users/" - logMsgToConfigSysLog "ERROR" "ERROR: Invalid Loggly username or password." + logMsgToConfigSysLog "ERROR" "ERROR: Invalid Loggly username or password. Your username is visible at the top right of the Loggly console before the @ symbol. You can reset your password at http://.loggly.com/login." exit 1 else logMsgToConfigSysLog "INFO" "INFO: Username and password authorized successfully." diff --git a/Mac Script/configure-mac.sh b/Mac Script/configure-mac.sh index b06c1f5..494a74a 100644 --- a/Mac Script/configure-mac.sh +++ b/Mac Script/configure-mac.sh @@ -248,7 +248,7 @@ checkIfValidUserNamePassword() echo "INFO: Checking if provided username and password is correct." if [ $(curl -s -u $LOGGLY_USERNAME:$LOGGLY_PASSWORD $LOGGLY_ACCOUNT_URL/apiv2/customer | grep "Unauthorized" | wc -l) == 1 ]; then logMsgToConfigSysLog "INFO" "INFO: Please check your username or reset your password at $LOGGLY_ACCOUNT_URL/account/users/" - logMsgToConfigSysLog "ERROR" "ERROR: Invalid Loggly username or password." + logMsgToConfigSysLog "ERROR" "ERROR: Invalid Loggly username or password. Your username is visible at the top right of the Loggly console before the @ symbol. You can reset your password at http://.loggly.com/login." exit 1 else logMsgToConfigSysLog "INFO" "INFO: Username and password authorized successfully." From 074e02a91cf62f0fb858905d0c00876d6a16fc7d Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Fri, 3 Apr 2015 23:18:22 +0530 Subject: [PATCH 48/53] Bug Fixes Keeping absolute path in the conf files --- Modular Scripts/File Monitoring/configure-file-monitoring.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Modular Scripts/File Monitoring/configure-file-monitoring.sh b/Modular Scripts/File Monitoring/configure-file-monitoring.sh index 04d0cd7..891faae 100644 --- a/Modular Scripts/File Monitoring/configure-file-monitoring.sh +++ b/Modular Scripts/File Monitoring/configure-file-monitoring.sh @@ -579,8 +579,7 @@ if [ "$1" != "being-invoked" ]; then LOGGLY_ROLLBACK="true" ;; -f | --filename ) shift - - LOGGLY_FILE_TO_MONITOR="${1%/}" + LOGGLY_FILE_TO_MONITOR="$(readlink -f ${1%/})" if [ -f "$LOGGLY_FILE_TO_MONITOR" ];then From 35cda1c8800e1de4e6f18e5cf57a944d6eff3cea Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Wed, 22 Apr 2015 12:11:54 +0530 Subject: [PATCH 49/53] version number update from 1.0 to 1.1 --- Mac Script/configure-mac.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Mac Script/configure-mac.sh b/Mac Script/configure-mac.sh index 494a74a..68fb64a 100644 --- a/Mac Script/configure-mac.sh +++ b/Mac Script/configure-mac.sh @@ -15,7 +15,7 @@ function ctrl_c() { #name of the current script. This will get overwritten by the child script which calls this SCRIPT_NAME=configure-mac.sh #version of the current script. This will get overwritten by the child script which calls this -SCRIPT_VERSION=1.0 +SCRIPT_VERSION=1.1 #application tag. This will get overwritten by the child script which calls this APP_TAG= From 9afecfa87ca986fea137901737562d827b1a4ecf Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Wed, 22 Apr 2015 13:47:42 +0530 Subject: [PATCH 50/53] version number update from 1.12 to 1.13 --- Linux Script/configure-linux.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Linux Script/configure-linux.sh b/Linux Script/configure-linux.sh index 13966fd..846bbdc 100644 --- a/Linux Script/configure-linux.sh +++ b/Linux Script/configure-linux.sh @@ -15,7 +15,7 @@ function ctrl_c() { #name of the current script. This will get overwritten by the child script which calls this SCRIPT_NAME=configure-linux.sh #version of the current script. This will get overwritten by the child script which calls this -SCRIPT_VERSION=1.12 +SCRIPT_VERSION=1.13 #application tag. This will get overwritten by the child script which calls this APP_TAG= From 271d3cddcf94f3ae824e83ccc4c4f3089e5fc780 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Wed, 22 Apr 2015 15:45:27 +0530 Subject: [PATCH 51/53] checks if cron is running during rollback --- .../configure-file-monitoring.sh | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/Modular Scripts/File Monitoring/configure-file-monitoring.sh b/Modular Scripts/File Monitoring/configure-file-monitoring.sh index 891faae..c372785 100644 --- a/Modular Scripts/File Monitoring/configure-file-monitoring.sh +++ b/Modular Scripts/File Monitoring/configure-file-monitoring.sh @@ -9,7 +9,7 @@ source configure-linux.sh "being-invoked" #name of the current script SCRIPT_NAME=configure-file-monitoring.sh #version of the current script -SCRIPT_VERSION=1.11 +SCRIPT_VERSION=1.12 #file to monitor (contains complete path and file name) provided by user LOGGLY_FILE_TO_MONITOR= @@ -125,14 +125,14 @@ removeLogglyConfForFile() #remove 21.conf file remove21ConfFile - + #restart rsyslog restartRsyslog removeStatFile #log success message - logMsgToConfigSysLog "INFO" "INFO: Rollback completed." + logMsgToConfigSysLog "SUCCESS" "SUCCESS: Rollback completed." } checkIfFileLocationContainSpaces() @@ -377,7 +377,7 @@ sudo cat << EOIPFW >> $CRON_SCRIPT $cronScriptStr EOIPFW - CRON_JOB_TO_MONITOR_FILES="*/5 * * * * sudo bash $CRON_SCRIPT" + CRON_JOB_TO_MONITOR_FILES="*/10 * * * * sudo bash $CRON_SCRIPT" CRON_FILE="/tmp/File_Monitor_Cron" EXISTING_CRONS=$(sudo crontab -l 2>&1) @@ -419,10 +419,8 @@ write21ConfFileContents() \$InputFileSeverity info \$InputFilePersistStateInterval 20000 \$InputRunFileMonitor - #Add a tag for file events \$template $CONF_FILE_FORMAT_NAME,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 $TAG] %msg%\n\" - if \$programname == '$LOGGLY_FILE_TO_MONITOR_ALIAS' then @@logs-01.loggly.com:514;$CONF_FILE_FORMAT_NAME if \$programname == '$LOGGLY_FILE_TO_MONITOR_ALIAS' then ~ " @@ -501,8 +499,13 @@ checkIfLogsAreParsedInLoggly() checkIfConfFileExist() { if [[ ! -f "$FILE_SYSLOG_CONFFILE" ]]; then - logMsgToConfigSysLog "ERROR" "ERROR: Invalid File Alias provided." - exit 1 + if [ $(sudo crontab -l 2>/dev/null | grep "file-monitoring-cron-$FILE_ALIAS.sh" | wc -l) -eq 1 ]; then + logMsgToConfigSysLog "ERROR" "ERROR: Cron is running to refresh configuration. Please try again after sometime." + exit 1 + else + logMsgToConfigSysLog "ERROR" "ERROR: Invalid File Alias provided." + exit 1 + fi fi } From 6da5daddb7f2a3c90318d76ed3ef737f0ad7e594 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Wed, 6 May 2015 12:43:57 +0530 Subject: [PATCH 52/53] Added script to publish on S3 --- publish.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 publish.sh diff --git a/publish.sh b/publish.sh new file mode 100644 index 0000000..3289a08 --- /dev/null +++ b/publish.sh @@ -0,0 +1,10 @@ +# Publishes the scripts to Loggly's S3 bucket where they are publically hosted. +# For Loggly's internal use only. Requires keys to publish. + +s3cmd put --acl-public Linux\ Script/configure-linux.sh s3://loggly-install/install/ +s3cmd put --acl-public Modular\ Scripts/File\ Monitoring/configure-file-monitoring.sh s3://loggly-install/install/ +s3cmd put --acl-public Modular\ Scripts/Apache2/configure-apache.sh s3://loggly-install/install/ +s3cmd put --acl-public Modular\ Scripts/Nginx/configure-nginx.sh s3://loggly-install/install/ +s3cmd put --acl-public Modular\ Scripts/S3Logs\ Monitoring/configure-s3-file-monitoring.sh s3://loggly-install/install/ +s3cmd put --acl-public Modular\ Scripts/Tomcat/configure-tomcat.sh s3://loggly-install/install/ +s3cmd put --acl-public Mac\ Script/configure-mac.sh s3://loggly-install/install/ From 4d7af771bc4ba73928f8940b2c3842a7f5d61d68 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Wed, 6 May 2015 13:49:02 +0530 Subject: [PATCH 53/53] Updated Manual instructions --- Mac Script/configure-mac.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Mac Script/configure-mac.sh b/Mac Script/configure-mac.sh index 68fb64a..65fe6a8 100644 --- a/Mac Script/configure-mac.sh +++ b/Mac Script/configure-mac.sh @@ -87,7 +87,7 @@ SUPPRESS_PROMPT="false" PROP_FILE= #manual instructions to be show in case of error -MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure rsyslog on Linux are available at https://www.loggly.com/docs/send-mac-logs-to-loggly/." +MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure Loggly on Mac are available at https://www.loggly.com/docs/send-mac-logs-to-loggly/." checkMacLogglyCompatibility() {