diff --git a/Linux Script/configure-linux.sh b/Linux Script/configure-linux.sh index 42ac9dd..846bbdc 100644 --- a/Linux Script/configure-linux.sh +++ b/Linux Script/configure-linux.sh @@ -15,7 +15,7 @@ function ctrl_c() { #name of the current script. This will get overwritten by the child script which calls this SCRIPT_NAME=configure-linux.sh #version of the current script. This will get overwritten by the child script which calls this -SCRIPT_VERSION=1.7 +SCRIPT_VERSION=1.13 #application tag. This will get overwritten by the child script which calls this APP_TAG= @@ -31,6 +31,8 @@ LOGGLY_RSYSLOG_CONFFILE_BACKUP=$LOGGLY_RSYSLOG_CONFFILE.loggly.bk RSYSLOG_DIR=/var/spool/rsyslog #rsyslog service name RSYSLOG_SERVICE=rsyslog +#syslog-ng +SYSLOG_NG_SERVICE=syslog-ng #rsyslogd RSYSLOGD=rsyslogd #minimum version of rsyslog to enable logging to loggly @@ -67,13 +69,16 @@ LOGGLY_USERNAME= #this is a mandatory input LOGGLY_PASSWORD= +#if this variable is set to true then suppress all prompts +SUPPRESS_PROMPT="false" + #variables used in 22-loggly.conf file LOGGLY_SYSLOG_PORT=514 LOGGLY_DISTRIBUTION_ID="41058" #Instruction link on how to configure loggly on linux manually. This will get overwritten by the child script which calls this #on how to configure the child application -MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure rsyslog on Linux are available at https://www.loggly.com/docs/rsyslog-manual-configuration/." +MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure rsyslog on Linux are available at https://www.loggly.com/docs/rsyslog-manual-configuration/. Rsyslog troubleshooting instructions are available at https://www.loggly.com/docs/troubleshooting-rsyslog/" #this variable is set if the script is invoked via some other calling script IS_INVOKED= @@ -111,6 +116,9 @@ checkLinuxLogglyCompatibility() #check if authentication token is valid. If no, then exit. checkIfValidAuthToken + #checking if syslog-ng is configured as a service + checkifSyslogNgConfiguredAsService + #check if rsyslog is configured as service. If no, then exit checkIfRsyslogConfiguredAsService @@ -137,7 +145,7 @@ installLogglyConf() fi #if all the above check passes, write the 22-loggly.conf file - write22LogglyConfFile + checkAuthTokenAndWriteContents #create rsyslog dir if it doesn't exist, Modify the permission on rsyslog directory if exist on Ubuntu createRsyslogDir @@ -205,27 +213,32 @@ checkIfSupportedOS() *"centos"* ) echo "INFO: Operating system is CentOS." ;; + *"debian"* ) + echo "INFO: Operating system is Debian." + ;; *"amazon"* ) echo "INFO: Operating system is Amazon AMI." ;; *"darwin"* ) #if the OS is mac then exit - logMsgToConfigSysLog "ERROR" "ERROR: This script is for Linux systems, and Darwin or Mac OSX are not currently supported. You can find alternative options here: https://www.loggly.com/docs" + logMsgToConfigSysLog "ERROR" "ERROR: This script is for Linux systems, and Darwin or Mac OSX are not currently supported. You can find alternative options here: https://www.loggly.com/docs/send-mac-logs-to-loggly/" exit 1 ;; * ) logMsgToConfigSysLog "WARN" "WARN: The linux distribution '$LINUX_DIST' has not been previously tested with Loggly." - while true; do - read -p "Would you like to continue anyway? (yes/no)" yn - case $yn in - [Yy]* ) - break;; - [Nn]* ) - exit 1 - ;; - * ) echo "Please answer yes or no.";; - esac - done + if [ "$SUPPRESS_PROMPT" == "false" ]; then + while true; do + read -p "Would you like to continue anyway? (yes/no)" yn + case $yn in + [Yy]* ) + break;; + [Nn]* ) + exit 1 + ;; + * ) echo "Please answer yes or no.";; + esac + done + fi ;; esac } @@ -306,7 +319,7 @@ checkIfValidUserNamePassword() echo "INFO: Checking if provided username and password is correct." if [ $(curl -s -u $LOGGLY_USERNAME:$LOGGLY_PASSWORD $LOGGLY_ACCOUNT_URL/apiv2/customer | grep "Unauthorized" | wc -l) == 1 ]; then logMsgToConfigSysLog "INFO" "INFO: Please check your username or reset your password at $LOGGLY_ACCOUNT_URL/account/users/" - logMsgToConfigSysLog "ERROR" "ERROR: Invalid Loggly username or password." + logMsgToConfigSysLog "ERROR" "ERROR: Invalid Loggly username or password. Your username is visible at the top right of the Loggly console before the @ symbol. You can reset your password at http://.loggly.com/login." exit 1 else logMsgToConfigSysLog "INFO" "INFO: Username and password authorized successfully." @@ -356,24 +369,34 @@ checkIfRsyslogConfiguredAsService() logMsgToConfigSysLog "ERROR" "ERROR: $RSYSLOG_SERVICE is not present as service." exit 1 fi - - if [ $(ps -ef | grep -v grep | grep "$RSYSLOG_SERVICE" | wc -l) -eq 0 ]; then + + #checking if syslog-ng is running as a service + checkifSyslogNgConfiguredAsService + + if [ $(ps -A | grep "$RSYSLOG_SERVICE" | wc -l) -eq 0 ]; then logMsgToConfigSysLog "INFO" "INFO: $RSYSLOG_SERVICE is not running. Attempting to start service." sudo service $RSYSLOG_SERVICE start fi } +checkifSyslogNgConfiguredAsService() +{ + if [ $(ps -A | grep "$SYSLOG_NG_SERVICE" | wc -l) -gt 0 ]; then + logMsgToConfigSysLog "ERROR" "ERROR: This script does not currently support syslog-ng. Please follow the instructions on this page https://www.loggly.com/docs/syslog-ng-manual-configuration" + exit 1 + fi +} #check if multiple versions of rsyslog is configured checkIfMultipleRsyslogConfigured() { - if [ $(ps -ef | grep -v grep | grep "$RSYSLOG_SERVICE" | wc -l) -gt 1 ]; then + if [ $(ps -A | grep "$RSYSLOG_SERVICE" | wc -l) -gt 1 ]; then logMsgToConfigSysLog "ERROR" "ERROR: Multiple (more than 1) $RSYSLOG_SERVICE is running." exit 1 fi } -#check if mimimum version of rsyslog required to configure loggly is met +#check if minimum version of rsyslog required to configure loggly is met checkIfMinVersionOfRsyslog() { RSYSLOG_VERSION=$(sudo $RSYSLOGD -version | grep "$RSYSLOGD") @@ -398,23 +421,10 @@ checkIfSelinuxServiceEnforced() fi } -#write 22-loggly,conf file to /etc/rsyslog.d directory after checking with user if override is needed -write22LogglyConfFile() -{ - echo "INFO: Checking if loggly sysconf file $LOGGLY_RSYSLOG_CONFFILE exist." - if [ -f "$LOGGLY_RSYSLOG_CONFFILE" ]; then - logMsgToConfigSysLog "INFO" "INFO: Loggly rsyslog file $LOGGLY_RSYSLOG_CONFFILE already exist." - checkIfConfigurationChanged - else - logMsgToConfigSysLog "INFO" "INFO: Loggly rsyslog file $LOGGLY_RSYSLOG_CONFFILE does not exist, creating file $LOGGLY_RSYSLOG_CONFFILE" - checkAuthTokenAndWriteContents - fi -} - #check if authentication token is valid and then write contents to 22-loggly.conf file to /etc/rsyslog.d directory checkAuthTokenAndWriteContents() { - if [ "$LOGGLY_ACCOUNT" != "" ]; then + if [ "$LOGGLY_AUTH_TOKEN" != "" ]; then writeContents $LOGGLY_ACCOUNT $LOGGLY_AUTH_TOKEN $LOGGLY_DISTRIBUTION_ID $LOGS_01_HOST $LOGGLY_SYSLOG_PORT restartRsyslog else @@ -423,69 +433,81 @@ checkAuthTokenAndWriteContents() fi } -#matches if the content of 22-loggly.conf content is changed -checkIfConfigurationChanged() -{ - ASK_FOR_VERIFICATION="false" - - #strings to be checked which should be present in the existing 22-loggly.conf. - #If these strings are not same then a warning message will be shown to user to update the 22-loggly.conf file - STR_TO_BE_CHECKED[0]="\$template LogglyFormat,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@$LOGGLY_DISTRIBUTION_ID] %msg%\"" - STR_TO_BE_CHECKED[1]="*.* @@$LOGS_01_HOST:$LOGGLY_SYSLOG_PORT;LogglyFormat" - - for i in "${STR_TO_BE_CHECKED[@]}" - do - if ! sudo grep -Fxq "$i" $LOGGLY_RSYSLOG_CONFFILE; then - ASK_FOR_VERIFICATION="true" - break; - fi - done - - if [ "$ASK_FOR_VERIFICATION" == "true" ]; then - logMsgToConfigSysLog "WARN" "WARN: Loggly rsyslog file /etc/rsyslog.d/22-loggly.conf content has changed." - while true; - do - read -p "Do you wish to override $LOGGLY_RSYSLOG_CONFFILE and re-verify configuration? (yes/no)" yn - case $yn in - [Yy]* ) - logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $LOGGLY_RSYSLOG_CONFFILE to $LOGGLY_RSYSLOG_CONFFILE_BACKUP"; - sudo mv -f $LOGGLY_RSYSLOG_CONFFILE $LOGGLY_RSYSLOG_CONFFILE_BACKUP; - checkAuthTokenAndWriteContents; - break;; - [Nn]* ) - LINUX_DO_VERIFICATION="false" - logMsgToConfigSysLog "INFO" "INFO: Skipping Linux verification." - break;; - * ) echo "Please answer yes or no.";; - esac - done - else - LINUX_DO_VERIFICATION="false" - fi - -} #write the contents to 22-loggly.conf file writeContents() { + +WRITE_SCRIPT_CONTENTS="false" inputStr=" # ------------------------------------------------------- # Syslog Logging Directives for Loggly ($1.loggly.com) # ------------------------------------------------------- # Define the template used for sending logs to Loggly. Do not change this format. -\$template LogglyFormat,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$2@$3] %msg%\" +\$template LogglyFormat,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$2@$3] %msg%\n\" + +\$WorkDirectory /var/spool/rsyslog # where to place spool files +\$ActionQueueFileName fwdRule1 # unique name prefix for spool files +\$ActionQueueMaxDiskSpace 1g # 1gb space limit (use as much as possible) +\$ActionQueueSaveOnShutdown on # save messages to disk on shutdown +\$ActionQueueType LinkedList # run asynchronously +\$ActionResumeRetryCount -1 # infinite retries if host is down # Send messages to Loggly over TCP using the template. *.* @@$4:$5;LogglyFormat -# ------------------------------------------------------- -# End of Syslog Logging Directives for Loggly -# ------------------------------------------------------- +# ------------------------------------------------------- " + if [ -f "$LOGGLY_RSYSLOG_CONFFILE" ]; then + logMsgToConfigSysLog "INFO" "INFO: Loggly rsyslog file $LOGGLY_RSYSLOG_CONFFILE already exist." + + STR_SIZE=${#inputStr} + SIZE_FILE=$(stat -c%s "$LOGGLY_RSYSLOG_CONFFILE") + + #actual file size and variable size with same contents always differ in size with one byte + STR_SIZE=$(( STR_SIZE + 1 )) + + if [ "$STR_SIZE" -ne "$SIZE_FILE" ]; then + + logMsgToConfigSysLog "WARN" "WARN: Loggly rsyslog file /etc/rsyslog.d/22-loggly.conf content has changed." + if [ "$SUPPRESS_PROMPT" == "false" ]; then + while true; + do + read -p "Do you wish to override $LOGGLY_RSYSLOG_CONFFILE and re-verify configuration? (yes/no)" yn + case $yn in + [Yy]* ) + logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $LOGGLY_RSYSLOG_CONFFILE to $LOGGLY_RSYSLOG_CONFFILE_BACKUP"; + sudo mv -f $LOGGLY_RSYSLOG_CONFFILE $LOGGLY_RSYSLOG_CONFFILE_BACKUP; + WRITE_SCRIPT_CONTENTS="true" + break;; + [Nn]* ) + LINUX_DO_VERIFICATION="false" + logMsgToConfigSysLog "INFO" "INFO: Skipping Linux verification." + break;; + * ) echo "Please answer yes or no.";; + esac + done + else + logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $LOGGLY_RSYSLOG_CONFFILE to $LOGGLY_RSYSLOG_CONFFILE_BACKUP"; + sudo mv -f $LOGGLY_RSYSLOG_CONFFILE $LOGGLY_RSYSLOG_CONFFILE_BACKUP; + WRITE_SCRIPT_CONTENTS="true" + fi + else + LINUX_DO_VERIFICATION="false" + fi + else + WRITE_SCRIPT_CONTENTS="true" + fi + + if [ "$WRITE_SCRIPT_CONTENTS" == "true" ]; then + sudo cat << EOIPFW >> $LOGGLY_RSYSLOG_CONFFILE $inputStr EOIPFW + + fi + } #create /var/spool/rsyslog directory if not already present. Modify the permission of this directory for Ubuntu @@ -535,17 +557,18 @@ checkIfLogsMadeToLoggly() searchAndFetch finalCount "$queryUrl" let counter=$counter+1 if [ "$counter" -gt "$maxCounter" ]; then - MANUAL_CONFIG_INSTRUCTION=$MANUAL_CONFIG_INSTRUCTION" Rsyslog troubleshooting instructions are available at https://www.loggly.com/docs/troubleshooting-rsyslog/" logMsgToConfigSysLog "ERROR" "ERROR: Logs did not make to Loggly in time. Please check network and firewall settings and retry." exit 1 fi done if [ "$finalCount" -eq 1 ]; then - logMsgToConfigSysLog "SUCCESS" "SUCCESS: Verification logs successfully transferred to Loggly! You are now sending Linux system logs to Loggly." if [ "$IS_INVOKED" = "" ]; then - exit 0 - fi + logMsgToConfigSysLog "SUCCESS" "SUCCESS: Verification logs successfully transferred to Loggly! You are now sending Linux system logs to Loggly." + exit 0 + else + logMsgToConfigSysLog "INFO" "SUCCESS: Verification logs successfully transferred to Loggly! You are now sending Linux system logs to Loggly." + fi fi } @@ -624,6 +647,7 @@ logMsgToConfigSysLog() #if it is a success, then log message "Script Succeeded" to config syslog and exit the script if [[ $cslStatus == "SUCCESS" ]]; then sendPayloadToConfigSysLog "SUCCESS" "Script Succeeded" "$enabler" + exit 0 fi } @@ -687,7 +711,7 @@ getPassword() usage() { cat << EOF -usage: configure-linux [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] +usage: configure-linux [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-s suppress prompts {optional)] usage: configure-linux [-a loggly auth account or subdomain] [-r to remove] usage: configure-linux [-h for help] EOF @@ -719,6 +743,9 @@ if [ "$1" != "being-invoked" ]; then -r | --remove ) LOGGLY_REMOVE="true" ;; + -s | --suppress ) + SUPPRESS_PROMPT="true" + ;; -h | --help) usage exit @@ -745,4 +772,6 @@ else IS_INVOKED="true" fi -########## Get Inputs from User - End ########## +########## Get Inputs from User - End ########## ------------------------------------------------------- +# End of Syslog Logging Directives for Loggly +# diff --git a/Mac Script/README.md b/Mac Script/README.md new file mode 100644 index 0000000..012519f --- /dev/null +++ b/Mac Script/README.md @@ -0,0 +1,10 @@ +Mac Script +============ + +Configure your Mac system to send syslogs to Loggly using the following command + + sudo bash configure-mac.sh -a SUBDOMAIN -u USERNAME + +Stop sending your Linux System logs to Loggly + + sudo bash configure-mac.sh -a SUBDOMAIN -r diff --git a/Mac Script/configure-mac.sh b/Mac Script/configure-mac.sh new file mode 100644 index 0000000..65fe6a8 --- /dev/null +++ b/Mac Script/configure-mac.sh @@ -0,0 +1,677 @@ +#!/bin/bash + +#trapping Control + C +#these statements must be the first statements in the script to trap the CTRL C event + +trap ctrl_c INT + +function ctrl_c() { + logMsgToConfigSysLog "INFO" "INFO: Aborting the script." + exit 1 +} + +########## Variable Declarations - Start ########## + +#name of the current script. This will get overwritten by the child script which calls this +SCRIPT_NAME=configure-mac.sh +#version of the current script. This will get overwritten by the child script which calls this +SCRIPT_VERSION=1.1 + +#application tag. This will get overwritten by the child script which calls this +APP_TAG= + +#this variable will hold the name of mac distribution +MAC_DIST= + +FLUENTD_CONF=loggly-fluentd.conf + +#ruby +RUBY=ruby + +#minimum version of ruby +MIN_RUBY_VERSION=1.9.3 + +#this variable will hold the users ruby version +RUBY_VERSION= + +#this variable will hold the host name +HOST_NAME= + +#host name for logs-01.loggly.com +LOGS_01_HOST=logs-01.loggly.com +LOGS_01_URL=https://$LOGS_01_HOST + +#this variable will contain loggly account url in the format https://$LOGGLY_ACCOUNT.loggly.com +LOGGLY_ACCOUNT_URL= + +#loggly.com URL +LOGGLY_COM_URL=https://www.loggly.com + +######Inputs provided by user###### +#this variable will hold the loggly account name provided by user. +#this is a mandatory input +LOGGLY_ACCOUNT= + +#this variable will hold the loggly authentication token provided by user. +#this is a mandatory input +LOGGLY_AUTH_TOKEN= + +#this variable will identify if the user has selected to rollback settings +LOGGLY_ROLLBACK= + +#this variable will hold the user name provided by user +#this is a mandatory input +LOGGLY_USERNAME= + +#this variable will hold the password provided by user +#this is a mandatory input +LOGGLY_PASSWORD= + +#variables used in fluentd.conf file +LOGGLY_SYSLOG_PORT=514 +LOGGLY_DISTRIBUTION_ID="41058" + +#this variable is set if the script is invoked via some other calling script +IS_INVOKED= + +#this variable will hold if the check env function for linux is invoked +MAC_ENV_VALIDATED="false" + +#this variable will inform if verification needs to be performed +MAC_DO_VERIFICATION="true" + +#if this variable is set to true then suppress all prompts +SUPPRESS_PROMPT="false" + +#plist file path +PROP_FILE= + +#manual instructions to be show in case of error +MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure Loggly on Mac are available at https://www.loggly.com/docs/send-mac-logs-to-loggly/." + +checkMacLogglyCompatibility() +{ + #check if the user has root permission to run this script + checkIfUserHasRootPrivileges + + #check if the OS is supported by the script. If no, then exit + checkIfSupportedOS + + #set the basic variables needed by this script + setMacVariables + + #check if the Loggly servers are accessible. If no, ask user to check network connectivity & exit + checkIfLogglyServersAccessible + + #check if user credentials are valid. If no, then exit + checkIfValidUserNamePassword + + #get authentication token if not provided + getAuthToken + + #check if authentication token is valid. If no, then exit. + checkIfValidAuthToken + + #check if minimum version of ruby is installed + checkIfMinRubyVersionInstalled + + MAC_ENV_VALIDATED="true" +} + +# executing the script for loggly to install and configure fluentd. +installLogglyConf() +{ + #log message indicating starting of Loggly configuration + logMsgToConfigSysLog "INFO" "INFO: Initiating Configure Loggly for Mac." + + if [ "$MAC_ENV_VALIDATED" = "false" ]; then + checkMacLogglyCompatibility + fi + + #checks if fluentd already installed otherwise installs it + checkIfFluentdInstalled + + #installs Loggly fluentd plugin + installLogglyFluentdPlugin + + #write conf file in the system + writeLogglyConfFile + + #configures Fluentd service + configureFluentdAsService + + #starts Fluentd service + startFluentdService + + if [ "$MAC_DO_VERIFICATION" = "true" ]; then + #check if the logs are going to loggly fro mac system now + checkIfLogsMadeToLoggly + fi + + if [ "$IS_INVOKED" = "" ]; then + logMsgToConfigSysLog "SUCCESS" "SUCCESS: Mac system successfully configured to send logs via Loggly." + fi +} + + +#remove loggly configuration from Linux system +removeLogglyConf() +{ + #log message indicating starting of Loggly configuration + logMsgToConfigSysLog "INFO" "INFO: Initiating uninstall Loggly for Mac." + + #check if the user has root permission to run this script + checkIfUserHasRootPrivileges + + #check if the OS is supported by the script. If no, then exit + checkIfSupportedOS + + #set the basic variables needed by this script + setMacVariables + + #remove loggly.conf file + removeLogglyConfFile + + + #log success message + logMsgToConfigSysLog "SUCCESS" "SUCCESS: Uninstalled Loggly configuration from Mac system." +} + +#checks if the script is run as a sudo user +checkIfUserHasRootPrivileges() +{ + #This script needs to be run as a sudo user + if [[ $EUID -ne 0 ]]; then + logMsgToConfigSysLog "ERROR" "ERROR: This script must be run as root." + exit 1 + fi +} + + +#check if supported operating system +checkIfSupportedOS() +{ + # Determine OS platform + UNAME=$(uname | tr "[:upper:]" "[:lower:]") + MAC_DIST=$UNAME + if [ "$MAC_DIST" == "darwin" ]; then + logMsgToConfigSysLog "INFO" "INFO: Operating system is Mac" + else + logMsgToConfigSysLog "ERROR": "ERROR: This script supports only Mac systems. You can find alternative options here: https://www.loggly.com/docs/sending-logs-unixlinux-system-setup/" + exit 1 + fi +} + + +#sets mac variables which will be used across various functions +setMacVariables() +{ + #set host name + HOST_NAME=$(hostname) + + #set loggly account url + LOGGLY_ACCOUNT_URL=https://$LOGGLY_ACCOUNT.loggly.com +} + + +#checks if all the various endpoints used for configuring loggly are accessible +checkIfLogglyServersAccessible() +{ + logMsgToConfigSysLog "INFO" "INFO: Checking if $LOGS_01_HOST is reachable." + if [ $(ping -c 1 $LOGS_01_HOST | grep "1 packets transmitted, 1 packets received, 0.0% packet loss" | wc -l) == 1 ]; then + logMsgToConfigSysLog "INFO" "INFO: $LOGS_01_HOST is reachable." + else + logMsgToConfigSysLog "ERROR" "ERROR: $LOGS_01_HOST is not reachable. Please check your network and firewall settings." + exit 1 + fi + + echo "INFO: Checking if $LOGS_01_HOST is reachable via $LOGGLY_SYSLOG_PORT port. This may take some time." + if [ $(curl --connect-timeout 10 $LOGS_01_HOST:$LOGGLY_SYSLOG_PORT 2>&1 | grep "Empty reply from server" | wc -l) == 1 ]; then + echo "INFO: $LOGS_01_HOST is reachable via $LOGGLY_SYSLOG_PORT port." + else + logMsgToConfigSysLog "ERROR" "ERROR: $LOGS_01_HOST is not reachable via $LOGGLY_SYSLOG_PORT port. Please check your network and firewall settings." + exit 1 + fi + + echo "INFO: Checking if '$LOGGLY_ACCOUNT' subdomain is valid." + if [ $(curl -s --head --request GET $LOGGLY_ACCOUNT_URL/login | grep "200 OK" | wc -l) == 1 ]; then + echo "INFO: $LOGGLY_ACCOUNT_URL is valid and reachable." + else + logMsgToConfigSysLog "ERROR" "ERROR: This is not a recognized subdomain. Please ask the account owner for the subdomain they signed up with." + exit 1 + fi +} + +#check if user name and password is valid +checkIfValidUserNamePassword() +{ + echo "INFO: Checking if provided username and password is correct." + if [ $(curl -s -u $LOGGLY_USERNAME:$LOGGLY_PASSWORD $LOGGLY_ACCOUNT_URL/apiv2/customer | grep "Unauthorized" | wc -l) == 1 ]; then + logMsgToConfigSysLog "INFO" "INFO: Please check your username or reset your password at $LOGGLY_ACCOUNT_URL/account/users/" + logMsgToConfigSysLog "ERROR" "ERROR: Invalid Loggly username or password. Your username is visible at the top right of the Loggly console before the @ symbol. You can reset your password at http://.loggly.com/login." + exit 1 + else + logMsgToConfigSysLog "INFO" "INFO: Username and password authorized successfully." + fi +} + +#gets the authentication token from the Loggly server +getAuthToken() +{ + if [ "$LOGGLY_AUTH_TOKEN" = "" ]; then + logMsgToConfigSysLog "INFO" "INFO: Authentication token not provided. Trying to retrieve it from $LOGGLY_ACCOUNT_URL account." + #get authentication token if user has not provided one + tokenstr=$(curl -s -u $LOGGLY_USERNAME:$LOGGLY_PASSWORD $LOGGLY_ACCOUNT_URL/apiv2/customer | grep -v "token") + + #get the string from index 0 to first occurence of , + tokenstr=${tokenstr%%,*} + + #get the string from index 0 to last occurence of " + tokenstr=${tokenstr%\"*} + + #get the string from first occurence of " to the end + tokenstr=${tokenstr#*\"} + + LOGGLY_AUTH_TOKEN=$tokenstr + + logMsgToConfigSysLog "INFO" "INFO: Retrieved authentication token: $LOGGLY_AUTH_TOKEN" + fi +} + + +#check if authentication token is valid +checkIfValidAuthToken() +{ + echo "INFO: Checking if provided auth token is correct." + if [ $(curl -s -u $LOGGLY_USERNAME:$LOGGLY_PASSWORD $LOGGLY_ACCOUNT_URL/apiv2/customer | grep \"$LOGGLY_AUTH_TOKEN\" | wc -l) == 1 ]; then + logMsgToConfigSysLog "INFO" "INFO: Authentication token validated successfully." + else + logMsgToConfigSysLog "ERROR" "ERROR: Invalid authentication token $LOGGLY_AUTH_TOKEN. You can get valid authentication token by following instructions at https://www.loggly.com/docs/customer-token-authentication-token/." + exit 1 + fi +} + +#this functions check if the min required version is installed in the system +checkIfMinRubyVersionInstalled() +{ + RUBY_VERSION=$(sudo $RUBY --version | grep "$RUBY") + RUBY_VERSION=${RUBY_VERSION%p*} + RUBY_VERSION=${RUBY_VERSION#* } + RUBY_VERSION=$RUBY_VERSION | tr -d " " + if [ $(compareVersions $RUBY_VERSION $MIN_RUBY_VERSION 3) -lt 0 ]; then + logMsgToConfigSysLog "ERROR" "ERROR: Min ruby version required is 1.9.3." + exit 1 + fi +} + +#this functions checks if the Fluentd gem is installed in the system +checkIfFluentdInstalled() +{ + if [ $(fluentd --setup ./fluent 2>/dev/null | grep "./fluent/fluent.conf" | wc -l ) == 1 ]; then + logMsgToConfigSysLog "INFO" "INFO: Fluentd is already installed. Not installing." + else + logMsgToConfigSysLog "INFO" "INFO: Fluentd is not installed. Installing Fluentd. This may take a while." + installFluentd + fi +} + +#this function installs the Fluentd in the system +installFluentd() +{ + #install fluentd gem http://docs.fluentd.org/articles/install-by-gem + sudo gem install fluentd --no-ri --no-rdoc + + #to check fluentd installed successfully + if [ $(fluentd --setup ./fluent 2>/dev/null | grep "./fluent/fluent.conf" | wc -l ) == 1 ]; then + logMsgToConfigSysLog "INFO" "INFO: Fluentd installed Successfully" + else + logMsgToConfigSysLog "ERROR" "ERROR: Unable to install fluentd" + exit 1 + fi +} + +#this function installs Loggly fluentd plugin +installLogglyFluentdPlugin() +{ + logMsgToConfigSysLog "INFO" "INFO: Installing Loggly plugin for Fluentd" + sudo gem install fluent-plugin-loggly + logMsgToConfigSysLog "INFO" "INFO: Loggly fluentd plugin installed successfully." +} + +#function to write the contents of fluentd config file +writeLogglyConfFile() +{ + + if [[ ! -d "$HOME/.loggly" ]]; then + mkdir $HOME/.loggly + fi + + FLUENTD_CONF="$HOME/.loggly/fluentd-loggly.conf" + + if [ -f "$FLUENTD_CONF" ]; then + echo "INFO: Conf file already exists. Creating Backup $FLUENTD_CONF $FLUENTD_CONF.bk" + sudo mv $FLUENTD_CONF $FLUENTD_CONF.bk + fi + + logMsgToConfigSysLog "INFO" "INFO: Creating file $FLUENTD_CONF" + + sudo touch $FLUENTD_CONF + +inputStr=" + + type tail + format none + path /var/log/system.log + tag system_logs + + + type loggly + loggly_url http://logs-01.loggly.com/inputs/$LOGGLY_AUTH_TOKEN/tag/Mac +" + +sudo cat << EOIPFW >> $FLUENTD_CONF +$inputStr +EOIPFW + +} + +#delete 22-loggly.conf file +removeLogglyConfFile() +{ + if [ -f "$HOME/.loggly/fluentd-loggly.conf" ]; then + logMsgToConfigSysLog "INFO" "INFO: Deleting file fluentd-loggly.conf" + sudo rm -rf "$HOME/.loggly/fluentd-loggly.conf" + + logMsgToConfigSysLog "INFO" "INFO: Removing Fluentd service" + sudo launchctl unload -F /Library/LaunchDaemons/com.loggly.loggly_fluentd.plist > /dev/null 2>&1 + sudo rm -rf /Library/LaunchDaemons/com.loggly.loggly_fluentd.plist + else + logMsgToConfigSysLog "ERROR" "ERROR: There is no conf file to delete" + exit 1 + fi +} + +#this function creates a fluentd daemon to send logs to Loggly +configureFluentdAsService() +{ + logMsgToConfigSysLog "INFO" "INFO: Creating daemon for Loggly conf file." + + #this sets the fluentd installation location + FLUENTD_LOCATION=$(which fluentd) + + PROP_FILE="/Library/LaunchDaemons/com.loggly.loggly_fluentd.plist" + + #if loggly fluentd is already running as a service then unload it + if [ $(sudo launchctl list | grep 'com.loggly.loggly_fluentd' | wc -l) == 1 ]; then + sudo launchctl unload -F $PROP_FILE > /dev/null 2>&1 + + #if there was some error while unloading, just remove it + sudo launchctl remove com.loggly.loggly_fluentd > /dev/null 2>&1 + fi + + #if plist file is already there then delete it + if [ -f "$PROP_FILE" ]; then + sudo rm -f $PROP_FILE + fi + + sudo touch $PROP_FILE + sudo chmod +x $PROP_FILE + +propStr=" + + + + Label + com.loggly.loggly_fluentd + ProgramArguments + + $FLUENTD_LOCATION + -c + $HOME/.loggly/fluentd-loggly.conf + + RunAtLoad + + StandardErrorPath + /tmp/loggly_fluentd.err + StandardOutPath + /tmp/loggly_fluentd.out + +" + +sudo cat << EOIPFW >> $PROP_FILE +$propStr +EOIPFW + +} + +#starts Fluentd Service +startFluentdService() +{ + logMsgToConfigSysLog "INFO" "INFO: Starting Fluentd as a service" + sudo launchctl load -F $PROP_FILE + logMsgToConfigSysLog "INFO" "INFO: Fluentd started successfully" +} + +#check if the logs made it to Loggly +checkIfLogsMadeToLoggly() +{ + logMsgToConfigSysLog "INFO" "INFO: Sending test message to Loggly." + uuid=$(cat /dev/urandom | env LC_CTYPE=C tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1) + + queryParam="tag%3AMac%20$uuid" + logger -t "Mac" "Mac-Test message for verification with UUID $uuid" + + counter=1 + maxCounter=10 + finalCount=0 + + queryUrl="$LOGGLY_ACCOUNT_URL/apiv2/search?q=$queryParam" + logMsgToConfigSysLog "INFO" "INFO: Search URL: $queryUrl" + + logMsgToConfigSysLog "INFO" "INFO: Verifying if the log made it to Loggly." + logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." + searchAndFetch finalCount "$queryUrl" + let counter=$counter+1 + + while [ "$finalCount" -eq 0 ]; do + echo "INFO: Did not find the test log message in Loggly's search yet. Waiting for 30 secs." + sleep 30 + echo "INFO: Done waiting. Verifying again." + logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." + searchAndFetch finalCount "$queryUrl" + let counter=$counter+1 + if [ "$counter" -gt "$maxCounter" ]; then + logMsgToConfigSysLog "ERROR" "ERROR: Logs did not make to Loggly in time. Please check network and firewall settings and retry." + exit 1 + fi + done + + if [ "$finalCount" -eq 1 ]; then + if [ "$IS_INVOKED" = "" ]; then + logMsgToConfigSysLog "SUCCESS" "SUCCESS: Verification logs successfully transferred to Loggly! You are now sending Mac system logs to Loggly." + exit 0 + else + logMsgToConfigSysLog "INFO" "SUCCESS: Verification logs successfully transferred to Loggly! You are now sending Mac system logs to Loggly." + fi + fi + +} + +compareVersions () +{ + typeset IFS='.' + typeset -a v1=( $1 ) + typeset -a v2=( $2 ) + typeset n diff + + for (( n=0; n<$3; n+=1 )); do + diff=$((v1[n]-v2[n])) + if [ $diff -ne 0 ] ; then + [ $diff -le 0 ] && echo '-1' || echo '1' + return + fi + done + echo '0' +} + + +#logs message to config syslog +logMsgToConfigSysLog() +{ + #$1 variable will be SUCCESS or ERROR or INFO or WARNING + #$2 variable will be the message + cslStatus=$1 + cslMessage=$2 + echo "$cslMessage" + currentTime=$(date) + + #for Linux system, we need to use -d switch to decode base64 whereas + #for Mac system, we need to use -D switch to decode + varUname=$(uname) + if [[ $varUname == 'Linux' ]]; then + enabler=$(echo -n MWVjNGU4ZTEtZmJiMi00N2U3LTkyOWItNzVhMWJmZjVmZmUw | base64 -d) + elif [[ $varUname == 'Darwin' ]]; then + enabler=$(echo MWVjNGU4ZTEtZmJiMi00N2U3LTkyOWItNzVhMWJmZjVmZmUw | base64 -D) + fi + + if [ $? -ne 0 ]; then + echo "ERROR: Base64 decode is not supported on your Operating System. Please update your system to support Base64." + exit 1 + fi + + sendPayloadToConfigSysLog "$cslStatus" "$cslMessage" "$enabler" + + #if it is an error, then log message "Script Failed" to config syslog and exit the script + if [[ $cslStatus == "ERROR" ]]; then + sendPayloadToConfigSysLog "ERROR" "Script Failed" "$enabler" + echo $MANUAL_CONFIG_INSTRUCTION + exit 1 + fi + + #if it is a success, then log message "Script Succeeded" to config syslog and exit the script + if [[ $cslStatus == "SUCCESS" ]]; then + sendPayloadToConfigSysLog "SUCCESS" "Script Succeeded" "$enabler" + exit 0 + fi +} + + +#payload construction to send log to config syslog +sendPayloadToConfigSysLog() +{ + if [ "$APP_TAG" = "" ]; then + var="{\"sub-domain\":\"$LOGGLY_ACCOUNT\", \"user-name\":\"$LOGGLY_USERNAME\", \"customer-token\":\"$LOGGLY_AUTH_TOKEN\", \"host-name\":\"$HOST_NAME\", \"script-name\":\"$SCRIPT_NAME\", \"script-version\":\"$SCRIPT_VERSION\", \"status\":\"$1\", \"time-stamp\":\"$currentTime\", \"Mac-distribution\":\"$MAC_DIST\", \"messages\":\"$2\",\"ruby-version\":\"$RUBY_VERSION\"}" + else + var="{\"sub-domain\":\"$LOGGLY_ACCOUNT\", \"user-name\":\"$LOGGLY_USERNAME\", \"customer-token\":\"$LOGGLY_AUTH_TOKEN\", \"host-name\":\"$HOST_NAME\", \"script-name\":\"$SCRIPT_NAME\", \"script-version\":\"$SCRIPT_VERSION\", \"status\":\"$1\", \"time-stamp\":\"$currentTime\", \"Mac-distribution\":\"$MAC_DIST\", $APP_TAG, \"messages\":\"$2\",\"ruby-version\":\"$RUBY_VERSION\"}" + fi + curl -s -H "content-type:application/json" -d "$var" $LOGS_01_URL/inputs/$3 > /dev/null 2>&1 +} + +#$1 return the count of records in loggly, $2 is the query param to search in loggly +searchAndFetch() +{ + url=$2 + + result=$(curl -s -u "$LOGGLY_USERNAME":"$LOGGLY_PASSWORD" "$url" ) + + if [ -z "$result" ]; then + logMsgToConfigSysLog "ERROR" "ERROR: Please check your network/firewall settings & ensure Loggly subdomain, username and password is specified correctly." + exit 1 + fi + id=$(echo "$result" | grep -v "{" | grep id | awk '{print $2}') + # strip last double quote from id + id="${id%\"}" + # strip first double quote from id + id="${id#\"}" + url="$LOGGLY_ACCOUNT_URL/apiv2/events?rsid=$id" + + # retrieve the data + result=$(curl -s -u "$LOGGLY_USERNAME":"$LOGGLY_PASSWORD" "$url" ) + count=$(echo "$result" | grep total_events | awk '{print $2}') + count="${count%\,}" + eval $1="'$count'" + if [ "$count" -gt 0 ]; then + timestamp=$(echo "$result" | grep timestamp) + fi +} + +#get password in the form of asterisk +getPassword() +{ + unset LOGGLY_PASSWORD + prompt="Please enter Loggly Password:" + while IFS= read -p "$prompt" -r -s -n 1 char + do + if [[ $char == $'\0' ]] + then + break + fi + prompt='*' + LOGGLY_PASSWORD+="$char" + done + echo +} + +#display usage syntax +usage() +{ +cat << EOF +usage: configure-mac [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] +usage: configure-mac [-a loggly auth account or subdomain] [-r to remove] +usage: configure-mac [-h for help] +EOF +} + +########## Get Inputs from User - Start ########## +if [ "$1" != "being-invoked" ]; then + if [ $# -eq 0 ]; then + usage + exit + else + while [ "$1" != "" ]; do + case $1 in + -t | --token ) shift + LOGGLY_AUTH_TOKEN=$1 + echo "AUTH TOKEN $LOGGLY_AUTH_TOKEN" + ;; + -a | --account ) shift + LOGGLY_ACCOUNT=$1 + echo "Loggly account or subdomain: $LOGGLY_ACCOUNT" + ;; + -u | --username ) shift + LOGGLY_USERNAME=$1 + echo "Username is set" + ;; + -p | --password ) shift + LOGGLY_PASSWORD=$1 + ;; + -r | --remove ) + LOGGLY_REMOVE="true" + ;; + -h | --help) + usage + exit + ;; + *) usage + exit + ;; + esac + shift + done + fi + + if [ "$LOGGLY_REMOVE" != "" -a "$LOGGLY_ACCOUNT" != "" ]; then + removeLogglyConf + elif [ "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_USERNAME" != "" ]; then + if [ "$LOGGLY_PASSWORD" = "" ]; then + getPassword + fi + installLogglyConf + else + usage + fi +else + IS_INVOKED="true" +fi + +########## Get Inputs from User - End ########## diff --git a/Modular Scripts/Apache2/configure-apache.sh b/Modular Scripts/Apache2/configure-apache.sh index e1ccf12..48f30c9 100644 --- a/Modular Scripts/Apache2/configure-apache.sh +++ b/Modular Scripts/Apache2/configure-apache.sh @@ -9,7 +9,7 @@ source configure-linux.sh "being-invoked" #name of the current script SCRIPT_NAME=configure-apache.sh #version of the current script -SCRIPT_VERSION=1.2 +SCRIPT_VERSION=1.5 #we have not found the apache version yet at this point in the script APP_TAG="\"apache-version\":\"\"" @@ -32,10 +32,16 @@ LOGGLY_APACHE_LOG_HOME= #this variable will hold the users apache version APACHE_VERSION= -MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure Apache2 is available at https://www.loggly.com/docs/sending-apache-logs/" +MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure Apache2 is available at https://www.loggly.com/docs/sending-apache-logs/. Rsyslog troubleshooting instructions are available at https://www.loggly.com/docs/troubleshooting-rsyslog/" #this variable will hold if the check env function for linux is invoked APACHE_ENV_VALIDATED="false" + +#apache as tag sent with the logs +LOGGLY_FILE_TAG="apache" + +#add tags to the logs +TAG= ########## Variable Declarations - End ########## #check if apache environment is compatible for Loggly @@ -64,7 +70,10 @@ installLogglyConfForApache() #configure loggly for Linux installLogglyConf - + + #multiple tags + addTagsInConfiguration + #create 21apache.conf file write21ApacheConfFile @@ -75,7 +84,7 @@ installLogglyConfForApache() checkIfApacheLogsMadeToLoggly #log success message - logMsgToConfigSysLog "SUCCESS" "SUCCESS: Apache successfully configured to send logs via Loggly." + logMsgToConfigSysLog "SUCCESS" "SUCCESS: Apache successfully configured to send logs to Loggly." } #executing script to remove loggly configuration for Apache @@ -172,20 +181,24 @@ checkLogFileSize() errorFileSize=$(wc -c "$2" | cut -f 1 -d ' ') fileSize=$((accessFileSize+errorFileSize)) if [ $fileSize -ge 102400000 ]; then - logMsgToConfigSysLog "INFO" "INFO: " - while true; do - read -p "WARN: There are currently large log files which may use up your allowed volume. Please rotate your logs before continuing. Would you like to continue now anyway? (yes/no)" yn - case $yn in - [Yy]* ) - logMsgToConfigSysLog "INFO" "INFO: Current apache logs size is $fileSize bytes. Continuing with Apache Loggly configuration."; - break;; - [Nn]* ) - logMsgToConfigSysLog "INFO" "INFO: Current apache logs size is $fileSize bytes. Discontinuing with Apache Loggly configuration." - exit 1 - break;; - * ) echo "Please answer yes or no.";; - esac - done + if [ "$SUPPRESS_PROMPT" == "false" ]; then + while true; do + read -p "WARN: There are currently large log files which may use up your allowed volume. Please rotate your logs before continuing. Would you like to continue now anyway? (yes/no)" yn + case $yn in + [Yy]* ) + logMsgToConfigSysLog "INFO" "INFO: Current apache logs size is $fileSize bytes. Continuing with Apache Loggly configuration."; + break;; + [Nn]* ) + logMsgToConfigSysLog "INFO" "INFO: Current apache logs size is $fileSize bytes. Discontinuing with Apache Loggly configuration." + exit 1 + break;; + * ) echo "Please answer yes or no.";; + esac + done + else + logMsgToConfigSysLog "WARN" "WARN: There are currently large log files which may use up your allowed volume." + logMsgToConfigSysLog "INFO" "INFO: Current apache logs size is $fileSize bytes. Continuing with Apache Loggly configuration." + fi elif [ $fileSize -eq 0 ]; then logMsgToConfigSysLog "WARN" "WARN: There are no recent logs from Apache there so won't be any sent to Loggly. You can generate some logs by visiting a page on your web server." exit 1 @@ -197,24 +210,40 @@ write21ApacheConfFile() #Create apache syslog config file if it doesn't exist echo "INFO: Checking if apache sysconf file $APACHE_SYSLOG_CONFFILE exist." if [ -f "$APACHE_SYSLOG_CONFFILE" ]; then + logMsgToConfigSysLog "WARN" "WARN: Apache syslog file $APACHE_SYSLOG_CONFFILE already exist." - while true; do - read -p "Do you wish to override $APACHE_SYSLOG_CONFFILE? (yes/no)" yn - case $yn in - [Yy]* ) - logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $APACHE_SYSLOG_CONFFILE to $APACHE_SYSLOG_CONFFILE_BACKUP"; - sudo mv -f $APACHE_SYSLOG_CONFFILE $APACHE_SYSLOG_CONFFILE_BACKUP; - write21ApacheFileContents; - break;; - [Nn]* ) break;; - * ) echo "Please answer yes or no.";; - esac - done + if [ "$SUPPRESS_PROMPT" == "false" ]; then + while true; do + read -p "Do you wish to override $APACHE_SYSLOG_CONFFILE? (yes/no)" yn + case $yn in + [Yy]* ) + logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $APACHE_SYSLOG_CONFFILE to $APACHE_SYSLOG_CONFFILE_BACKUP"; + sudo mv -f $APACHE_SYSLOG_CONFFILE $APACHE_SYSLOG_CONFFILE_BACKUP; + write21ApacheFileContents; + break;; + [Nn]* ) break;; + * ) echo "Please answer yes or no.";; + esac + done + else + logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $APACHE_SYSLOG_CONFFILE to $APACHE_SYSLOG_CONFFILE_BACKUP"; + sudo mv -f $APACHE_SYSLOG_CONFFILE $APACHE_SYSLOG_CONFFILE_BACKUP; + write21ApacheFileContents; + fi else write21ApacheFileContents fi } +addTagsInConfiguration() +{ + #split tags by comman(,) + IFS=, read -a array <<< "$LOGGLY_FILE_TAG" + for i in "${array[@]}" + do + TAG="$TAG tag=\\\"$i\\\" " + done +} #function to write the contents of apache syslog config file write21ApacheFileContents() { @@ -249,7 +278,7 @@ write21ApacheFileContents() \$InputRunFileMonitor #Add a tag for apache events - \$template LogglyFormatApache,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 tag=\\\"apache\\\"] %msg%\n\" + \$template LogglyFormatApache,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 $TAG] %msg%\n\" if \$programname == 'apache-access' then @@logs-01.loggly.com:514;LogglyFormatApache if \$programname == 'apache-access' then ~ @@ -274,8 +303,19 @@ checkIfApacheLogsMadeToLoggly() apacheInitialLogCount=0 apacheLatestLogCount=0 - queryParam="tag%3Aapache&from=-15m&until=now&size=1" - + + TAGS= + IFS=, read -a array <<< "$LOGGLY_FILE_TAG" + for i in "${array[@]}" + do + if [ "$TAGS" == "" ]; then + TAGS="tag%3A$i" + else + TAGS="$TAGS%20tag%3A$i" + fi + done + + queryParam="$TAGS&from=-15m&until=now&size=1" queryUrl="$LOGGLY_ACCOUNT_URL/apiv2/search?q=$queryParam" logMsgToConfigSysLog "INFO" "INFO: Search URL: $queryUrl" @@ -297,14 +337,35 @@ checkIfApacheLogsMadeToLoggly() searchAndFetch apacheLatestLogCount "$queryUrl" let counter=$counter+1 if [ "$counter" -gt "$maxCounter" ]; then - logMsgToConfigSysLog "ERROR" "ERROR: Apache logs did not make to Loggly in time. Please check network and firewall settings and retry." - exit 1 + logMsgToConfigSysLog "ERROR" "ERROR: Apache logs did not make to Loggly in time. Please check network and firewall settings and retry." + exit 1 fi done if [ "$apacheLatestLogCount" -gt "$apacheInitialLogCount" ]; then - logMsgToConfigSysLog "SUCCESS" "SUCCESS: Apache logs successfully transferred to Loggly! You are now sending Apache logs to Loggly." - exit 0 + logMsgToConfigSysLog "INFO" "INFO: Apache logs successfully transferred to Loggly! You are now sending Apache logs to Loggly." + checkIfLogsAreParsedInLoggly + fi +} +#verifying if the logs are being parsed or not +checkIfLogsAreParsedInLoggly() +{ + apacheInitialLogCount=0 + TAG_PARSER= + IFS=, read -a array <<< "$LOGGLY_FILE_TAG" + + for i in "${array[@]}" + do + TAG_PARSER="$TAG_PARSER%20tag%3A$i " + done + queryParam="logtype%3Aapache$TAG_PARSER&from=-15m&until=now&size=1" + queryUrl="$LOGGLY_ACCOUNT_URL/apiv2/search?q=$queryParam" + searchAndFetch apacheInitialLogCount "$queryUrl" + logMsgToConfigSysLog "INFO" "INFO: Verifying if the Apache logs are parsed in Loggly." + if [ "$apacheInitialLogCount" -gt 0 ]; then + logMsgToConfigSysLog "INFO" "INFO: Apache logs successfully parsed in Loggly!" + else + logMsgToConfigSysLog "WARN" "WARN: We received your logs but they do not appear to use one of our automatically parsed formats. You can still do full text search and counts on these logs, but you won't be able to use our field explorer. Please consider switching to one of our automated formats https://www.loggly.com/docs/automated-parsing/" fi } @@ -323,7 +384,7 @@ remove21ApacheConfFile() usage() { cat << EOF -usage: configure-apache [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] +usage: configure-apache [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-tag filetag1,filetag2 (optional)] [-s suppress prompts {optional)] usage: configure-apache [-a loggly auth account or subdomain] [-r to rollback] usage: configure-apache [-h for help] EOF @@ -352,9 +413,16 @@ while [ "$1" != "" ]; do -p | --password ) shift LOGGLY_PASSWORD=$1 ;; + -tag| --filetag ) shift + LOGGLY_FILE_TAG=$1 + echo "File tag: $LOGGLY_FILE_TAG" + ;; -r | --rollback ) LOGGLY_ROLLBACK="true" ;; + -s | --suppress ) + SUPPRESS_PROMPT="true" + ;; -h | --help) usage exit diff --git a/Modular Scripts/File Monitoring/configure-file-monitoring.sh b/Modular Scripts/File Monitoring/configure-file-monitoring.sh index 18f3fce..c372785 100644 --- a/Modular Scripts/File Monitoring/configure-file-monitoring.sh +++ b/Modular Scripts/File Monitoring/configure-file-monitoring.sh @@ -9,13 +9,15 @@ source configure-linux.sh "being-invoked" #name of the current script SCRIPT_NAME=configure-file-monitoring.sh #version of the current script -SCRIPT_VERSION=1.5 +SCRIPT_VERSION=1.12 #file to monitor (contains complete path and file name) provided by user LOGGLY_FILE_TO_MONITOR= #alias name, will be used as tag & state file name etc. provided by user LOGGLY_FILE_TO_MONITOR_ALIAS= +FILE_ALIAS= +STATE_FILE_ALIAS= #file alias provided by the user APP_TAG="\"file-alias\":\"\"" @@ -26,7 +28,7 @@ FILE_SYSLOG_CONFFILE= #name and location of syslog backup file FILE_SYSLOG_CONFFILE_BACKUP= -MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure a file is available at https://www.loggly.com/docs/file-monitoring/" +MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure a file is available at https://www.loggly.com/docs/file-monitoring/. Rsyslog troubleshooting instructions are available at https://www.loggly.com/docs/troubleshooting-rsyslog/" #this variable is set if the script is invoked via some other calling script IS_FILE_MONITOR_SCRIPT_INVOKED="false" @@ -37,6 +39,14 @@ LOGGLY_FILE_TAG="file" #format name for the conf file. Can be set by calling script CONF_FILE_FORMAT_NAME="LogglyFormatFile" +#add tags to the logs +TAG= + +FILE_TO_MONITOR= + +IS_DIRECTORY= + +IS_WILDCARD= ########## Variable Declarations - End ########## # executing the script for loggly to install and configure syslog @@ -50,36 +60,50 @@ installLogglyConfForFile() #checks if the file name contain spaces, if yes, the exit checkIfFileLocationContainSpaces - - #construct variables using filename and filealias - constructFileVariables - - #check if file to monitor exists - checkIfFileExist - - #checks if the file has proper read permission - checkFileReadPermission #check if the alias is already taken checkIfFileAliasExist - - #configure loggly for Linux - installLogglyConf - - #create 21.conf file - write21ConfFileContents - + + if [ "$IS_DIRECTORY" == "true" ]; then + + configureDirectoryFileMonitoring + + else + + #check if file to monitor exists + checkIfFileExist + + #construct variables using filename and filealias + constructFileVariables + + #check for the log file size + checkLogFileSize $LOGGLY_FILE_TO_MONITOR + + #checks if the file has proper read permission + checkFileReadPermission + + #configure loggly for Linux + installLogglyConf + + #multiple tags + addTagsInConfiguration + + #create 21.conf file + write21ConfFileContents + + fi + #restart rsyslog restartRsyslog - - #check for the log file size - checkLogFileSize $LOGGLY_FILE_TO_MONITOR #verify if the file logs made it to loggly checkIfFileLogsMadeToLoggly - - #log success message - logMsgToConfigSysLog "SUCCESS" "SUCCESS: Successfully configured to send $LOGGLY_FILE_TO_MONITOR logs via Loggly." + + + if [ "$IS_FILE_MONITOR_SCRIPT_INVOKED" = "false" ]; then + #log success message + logMsgToConfigSysLog "SUCCESS" "SUCCESS: Successfully configured to send $LOGGLY_FILE_TO_MONITOR logs via Loggly." + fi } #executing script to remove loggly configuration for File @@ -101,12 +125,14 @@ removeLogglyConfForFile() #remove 21.conf file remove21ConfFile - + #restart rsyslog restartRsyslog + removeStatFile + #log success message - logMsgToConfigSysLog "INFO" "INFO: Rollback completed." + logMsgToConfigSysLog "SUCCESS" "SUCCESS: Rollback completed." } checkIfFileLocationContainSpaces() @@ -122,15 +148,100 @@ checkIfFileLocationContainSpaces() constructFileVariables() { #conf file name - FILE_SYSLOG_CONFFILE="$RSYSLOG_ETCDIR_CONF/21-filemonitoring-$LOGGLY_FILE_TO_MONITOR_ALIAS.conf" + FILE_SYSLOG_CONFFILE="$RSYSLOG_ETCDIR_CONF/21-filemonitoring-$FILE_ALIAS.conf" #conf file backup name - FILE_SYSLOG_CONFFILE_BACKUP="$FILE_SYSLOG_CONFFILE.loggly.bk" + FILE_SYSLOG_CONFFILE_BACKUP="$FILE_ALIAS.loggly.bk" #application tag APP_TAG="\"file-alias\":\"$LOGGLY_FILE_TO_MONITOR_ALIAS\"" } +#configures the directory files for file monitoring +configureDirectoryFileMonitoring() +{ + addTagsInConfiguration + TOTAL_FILES_IN_DIR=$(ls -1 ${LOGGLY_FILE_TO_MONITOR} | wc -l) + logMsgToConfigSysLog "INFO" "INFO: There are $TOTAL_FILES_IN_DIR files in directory. Configuring each file for monitoring present in this directory." + if [ "$SUPPRESS_PROMPT" == "false" ]; then + while true; do + read -p "There are $TOTAL_FILES_IN_DIR files present in this directory. Would you like to configure all the files (yes/no)?" yn + case $yn in + [Yy]* ) + installLogglyConf + for file in $(find $LOGGLY_FILE_TO_MONITOR -name '*') + do + configureFilesPresentInDirectory $file $FILE_ALIAS + done + break;; + [Nn]* ) + exit 1 + break;; + * ) echo "Please answer yes or no.";; + esac + done + while true; do + read -p "Would you like install a Cron job to sync the files currently in your Directory every 5 minutes? (yes/no)" yn + case $yn in + [Yy]* ) + doCronInstallation + break;; + [Nn]* ) + logMsgToConfigSysLog "INFO" "INFO: Skipping Cron installation." + break;; + * ) echo "Please answer yes or no.";; + esac + done + else + installLogglyConf + for file in $(find $LOGGLY_FILE_TO_MONITOR -name '*') + do + configureFilesPresentInDirectory $file $FILE_ALIAS + done + if [[ ! -f "/root/.loggly/file-monitoring-cron-$FILE_ALIAS.sh" ]]; then + doCronInstallation + fi + fi +} + +#actually configures a file present in the directory for monitoring +configureFilesPresentInDirectory() +{ + FILE_TO_MONITOR=$1 + fileNameWithExt=${1##*/} + uniqueFileName=$(echo "$fileNameWithExt" | tr . _) + var=$(file $FILE_TO_MONITOR) + + #checking if it is a text file otherwise ignore it + #it may be possible that the "text" may contain some uppercase letters like "Text" + var=$(echo $var | tr "[:upper:]" "[:lower:]") + if [[ $var == *text* ]]; then + LOGGLY_FILE_TO_MONITOR_ALIAS=$uniqueFileName-$2 + if [ -f ${FILE_TO_MONITOR} ]; then + constructFileVariables + checkFileReadPermission + checkLogFileSize $FILE_TO_MONITOR + STATE_FILE_ALIAS=$(echo -n "$uniqueFileName" | md5sum | tr -d ' ')$FILE_ALIAS + write21ConfFileContents + fi + fi +} + +checkIfWildcardExist() +{ + TOTAL_FILES_IN_DIR=$(ls -1 ${LOGGLY_FILE_TO_MONITOR} 2> /dev/null | wc -l ) + WILDCARDS=( '*' '.' '?' '|' ']' '[' ) + for WILDCARD in "${WILDCARDS[@]}"; + do + if [[ $LOGGLY_FILE_TO_MONITOR == *"${WILDCARD}"* && $TOTAL_FILES_IN_DIR -gt 0 ]]; then + IS_WILDCARD="true" + return 0 + else + return 1 + fi + done +} + #checks if the file to be monitored exist checkIfFileExist() { @@ -142,25 +253,40 @@ checkIfFileExist() fi } +#deletes the state file for the current alias, if exists +deleteStateFile() +{ + restartRsyslog + sudo rm -f $RSYSLOG_DIR/stat-$FILE_ALIAS + restartRsyslog +} + #check if the file alias is already taken checkIfFileAliasExist() { if [ -f "$FILE_SYSLOG_CONFFILE" ]; then logMsgToConfigSysLog "WARN" "WARN: This file alias is already taken. You must choose a unique file alias for each file." - while true; do - read -p "Would you like to overwrite the configuration for this file alias (yes/no)?" yn - case $yn in - [Yy]* ) - logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $FILE_SYSLOG_CONFFILE to $FILE_SYSLOG_CONFFILE_BACKUP"; - sudo mv -f $FILE_SYSLOG_CONFFILE $FILE_SYSLOG_CONFFILE_BACKUP; - break;; - [Nn]* ) - logMsgToConfigSysLog "INFO" "INFO: Not overwriting the existing configuration. Exiting" - exit 1 - break;; - * ) echo "Please answer yes or no.";; - esac - done + if [ "$SUPPRESS_PROMPT" == "false" ]; then + while true; do + read -p "Would you like to overwrite the configuration for this file alias (yes/no)?" yn + case $yn in + [Yy]* ) + logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $FILE_SYSLOG_CONFFILE to $FILE_SYSLOG_CONFFILE_BACKUP"; + sudo mv -f $FILE_SYSLOG_CONFFILE $FILE_SYSLOG_CONFFILE_BACKUP + deleteStateFile + break;; + [Nn]* ) + logMsgToConfigSysLog "INFO" "INFO: Not overwriting the existing configuration. Exiting" + exit 1 + break;; + * ) echo "Please answer yes or no.";; + esac + done + else + logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $FILE_SYSLOG_CONFFILE to $FILE_SYSLOG_CONFFILE_BACKUP"; + sudo mv -f $FILE_SYSLOG_CONFFILE $FILE_SYSLOG_CONFFILE_BACKUP + deleteStateFile + fi fi } @@ -170,35 +296,37 @@ checkLogFileSize() { monitorFileSize=$(wc -c "$1" | cut -f 1 -d ' ') if [ $monitorFileSize -ge 102400000 ]; then - logMsgToConfigSysLog "INFO" "INFO: " - while true; do - read -p "WARN: There are currently large log files which may use up your allowed volume. Please rotate your logs before continuing. Would you like to continue now anyway? (yes/no)" yn - case $yn in - [Yy]* ) - logMsgToConfigSysLog "INFO" "INFO: Current size of $LOGGLY_FILE_TO_MONITOR is $monitorFileSize bytes. Continuing with File Loggly configuration."; - break;; - [Nn]* ) - logMsgToConfigSysLog "INFO" "INFO: Current size of $LOGGLY_FILE_TO_MONITOR is $monitorFileSize bytes. Discontinuing with File Loggly configuration." - exit 1 - break;; - * ) echo "Please answer yes or no.";; - esac - done + if [ "$SUPPRESS_PROMPT" == "false" ]; then + while true; do + read -p "WARN: There are currently large log files which may use up your allowed volume. Please rotate your logs before continuing. Would you like to continue now anyway? (yes/no)" yn + case $yn in + [Yy]* ) + logMsgToConfigSysLog "INFO" "INFO: Current size of $LOGGLY_FILE_TO_MONITOR is $monitorFileSize bytes. Continuing with File Loggly configuration."; + break;; + [Nn]* ) + logMsgToConfigSysLog "INFO" "INFO: Current size of $LOGGLY_FILE_TO_MONITOR is $monitorFileSize bytes. Discontinuing with File Loggly configuration." + exit 1 + break;; + * ) echo "Please answer yes or no.";; + esac + done + else + logMsgToConfigSysLog "WARN" "WARN: There are currently large log files which may use up your allowed volume." + logMsgToConfigSysLog "INFO" "INFO: Current size of $LOGGLY_FILE_TO_MONITOR is $monitorFileSize bytes. Continuing with File Loggly configuration."; + fi elif [ $monitorFileSize -eq 0 ]; then logMsgToConfigSysLog "WARN" "WARN: There are no recent logs from $LOGGLY_FILE_TO_MONITOR so there won't be any data sent to Loggly. You can generate some logs by writing to this file." exit 1 else - logMsgToConfigSysLog "INFO" "INFO: File size of $LOGGLY_FILE_TO_MONITOR is $monitorFileSize bytes." + logMsgToConfigSysLog "INFO" "INFO: File size of $FILE_TO_MONITOR is $monitorFileSize bytes." fi } #checks the input file has proper read permissions checkFileReadPermission() -{ - +{ LINUX_DIST_IN_LOWER_CASE=$(echo $LINUX_DIST | tr "[:upper:]" "[:lower:]") - #no need to check read permissions with RedHat and CentOS as they also work with ---------- (000)permissions case "$LINUX_DIST_IN_LOWER_CASE" in *"redhat"* ) @@ -209,12 +337,61 @@ checkFileReadPermission() FILE_PERMISSIONS=$(ls -l $LOGGLY_FILE_TO_MONITOR) #checking if the file has read permission for others PERMISSION_READ_OTHERS=${FILE_PERMISSIONS:7:1} - if [ $PERMISSION_READ_OTHERS != r ]; then + if [[ $PERMISSION_READ_OTHERS != r ]]; then logMsgToConfigSysLog "WARN" "WARN: $LOGGLY_FILE_TO_MONITOR does not have proper read permissions. Verification step may fail." fi ;; esac +} + +addTagsInConfiguration() +{ + #split tags by comman(,) + IFS=, read -a array <<< "$LOGGLY_FILE_TAG" + for i in "${array[@]}" + do + TAG="$TAG tag=\\\"$i\\\" " + done +} + +doCronInstallation() +{ + if [[ ! -d "/root/.loggly" ]]; then + mkdir /root/.loggly + fi + CRON_SCRIPT="/root/.loggly/file-monitoring-cron-$FILE_ALIAS.sh" + logMsgToConfigSysLog "INFO" "INFO: Creating cron script $CRON_SCRIPT" + +sudo touch $CRON_SCRIPT +sudo chmod +x $CRON_SCRIPT + +cronScriptStr="#!/bin/bash +curl -s -o configure-file-monitoring.sh https://www.loggly.com/install/configure-file-monitoring.sh +sudo mv -f $FILE_SYSLOG_CONFFILE $FILE_SYSLOG_CONFFILE.bk +sudo rm -f $FILE_SYSLOG_CONFFILE +sudo bash configure-file-monitoring.sh -a $LOGGLY_ACCOUNT -u $LOGGLY_USERNAME -p $LOGGLY_PASSWORD -f $LOGGLY_FILE_TO_MONITOR -l $FILE_ALIAS -tag $LOGGLY_FILE_TAG -s +" +#write to cron script file + +sudo cat << EOIPFW >> $CRON_SCRIPT +$cronScriptStr +EOIPFW + + CRON_JOB_TO_MONITOR_FILES="*/10 * * * * sudo bash $CRON_SCRIPT" + CRON_FILE="/tmp/File_Monitor_Cron" + + EXISTING_CRONS=$(sudo crontab -l 2>&1) + case $EXISTING_CRONS in + no*) + ;; + *) + echo "$EXISTING_CRONS" >> $CRON_FILE + ;; + esac + echo "$CRON_JOB_TO_MONITOR_FILES" >> $CRON_FILE + sudo crontab $CRON_FILE + sudo rm -fr $CRON_FILE } #function to write the contents of syslog config file @@ -224,7 +401,8 @@ write21ConfFileContents() sudo touch $FILE_SYSLOG_CONFFILE sudo chmod o+w $FILE_SYSLOG_CONFFILE - imfileStr="\$ModLoad imfile + imfileStr=" + \$ModLoad imfile \$InputFilePollInterval 10 \$WorkDirectory $RSYSLOG_DIR " @@ -232,19 +410,17 @@ write21ConfFileContents() imfileStr+="\$PrivDropToGroup adm " fi - + imfileStr+=" # File access file: - \$InputFileName $LOGGLY_FILE_TO_MONITOR + \$InputFileName $FILE_TO_MONITOR \$InputFileTag $LOGGLY_FILE_TO_MONITOR_ALIAS: - \$InputFileStateFile stat-$LOGGLY_FILE_TO_MONITOR_ALIAS + \$InputFileStateFile stat-$STATE_FILE_ALIAS \$InputFileSeverity info \$InputFilePersistStateInterval 20000 \$InputRunFileMonitor - #Add a tag for file events - \$template $CONF_FILE_FORMAT_NAME,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 tag=\\\"$LOGGLY_FILE_TAG\\\"] %msg%\n\" - + \$template $CONF_FILE_FORMAT_NAME,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 $TAG] %msg%\n\" if \$programname == '$LOGGLY_FILE_TO_MONITOR_ALIAS' then @@logs-01.loggly.com:514;$CONF_FILE_FORMAT_NAME if \$programname == '$LOGGLY_FILE_TO_MONITOR_ALIAS' then ~ " @@ -293,19 +469,43 @@ checkIfFileLogsMadeToLoggly() done if [ "$fileLatestLogCount" -gt "$fileInitialLogCount" ]; then - logMsgToConfigSysLog "SUCCESS" "SUCCESS: Logs successfully transferred to Loggly! You are now sending $LOGGLY_FILE_TO_MONITOR logs to Loggly." - if [ "$IS_FILE_MONITOR_SCRIPT_INVOKED" = "false" ]; then - exit 0 - fi + logMsgToConfigSysLog "INFO" "INFO: Logs successfully transferred to Loggly! You are now sending $LOGGLY_FILE_TO_MONITOR logs to Loggly." + checkIfLogsAreParsedInLoggly + fi +} + +#verifying if the logs are being parsed or not +checkIfLogsAreParsedInLoggly() +{ + fileInitialLogCount=0 + TAG_PARSER= + IFS=, read -a array <<< "$LOGGLY_FILE_TAG" + for i in "${array[@]}" + do + TAG_PARSER="$TAG_PARSER%20tag%3A$i " + done + + queryParam="syslog.appName%3A$LOGGLY_FILE_TO_MONITOR_ALIAS$TAG_PARSER&from=-15m&until=now&size=1" + queryUrl="$LOGGLY_ACCOUNT_URL/apiv2/search?q=$queryParam" + searchAndFetch fileInitialLogCount "$queryUrl" + if [ "$fileInitialLogCount" -gt 0 ]; then + logMsgToConfigSysLog "INFO" "INFO: File logs successfully parsed in Loggly!" + else + logMsgToConfigSysLog "WARN" "WARN: We received your logs but they do not appear to use one of our automatically parsed formats. You can still do full text search and counts on these logs, but you won't be able to use our field explorer. Please consider switching to one of our automated formats https://www.loggly.com/docs/automated-parsing/" fi } #checks if the conf file exist. Name of conf file is constructed using the file alias name provided checkIfConfFileExist() { - if [ ! -f "$FILE_SYSLOG_CONFFILE" ]; then - logMsgToConfigSysLog "ERROR" "ERROR: Invalid File Alias provided." - exit 1 + if [[ ! -f "$FILE_SYSLOG_CONFFILE" ]]; then + if [ $(sudo crontab -l 2>/dev/null | grep "file-monitoring-cron-$FILE_ALIAS.sh" | wc -l) -eq 1 ]; then + logMsgToConfigSysLog "ERROR" "ERROR: Cron is running to refresh configuration. Please try again after sometime." + exit 1 + else + logMsgToConfigSysLog "ERROR" "ERROR: Invalid File Alias provided." + exit 1 + fi fi } @@ -315,6 +515,7 @@ remove21ConfFile() echo "INFO: Deleting the loggly syslog conf file $FILE_SYSLOG_CONFFILE." if [ -f "$FILE_SYSLOG_CONFFILE" ]; then sudo rm -rf "$FILE_SYSLOG_CONFFILE" + deleteFileFromCrontab if [ "$IS_FILE_MONITOR_SCRIPT_INVOKED" = "false" ]; then echo "INFO: Removed all the modified files." fi @@ -323,11 +524,32 @@ remove21ConfFile() fi } +deleteFileFromCrontab() +{ + if [ -f "/root/.loggly/file-monitoring-cron-$FILE_ALIAS.sh" ];then + + logMsgToConfigSysLog "INFO" "INFO: Deleting sync Cron." + + #delete cron + sudo crontab -l | grep -v "file-monitoring-cron-$FILE_ALIAS.sh" | crontab - + + #delete cron script + sudo rm -f /root/.loggly/file-monitoring-cron-$FILE_ALIAS.sh + + fi + +} + +removeStatFile() +{ + sudo rm -f $RSYSLOG_DIR/stat-*$FILE_ALIAS +} + #display usage syntax usage() { cat << EOF -usage: configure-file-monitoring [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-f filename] [-tag filetag (optional)] [-l filealias] +usage: configure-file-monitoring [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-f filename] [-tag filetag1,filetag2 (optional)] [-l filealias] [-s suppress prompts {optional)] usage: configure-file-monitoring [-a loggly auth account or subdomain] [-r to rollback] [-l filealias] usage: configure-file-monitoring [-h for help] EOF @@ -360,12 +582,28 @@ if [ "$1" != "being-invoked" ]; then LOGGLY_ROLLBACK="true" ;; -f | --filename ) shift - #LOGGLY_FILE_TO_MONITOR=$1 - LOGGLY_FILE_TO_MONITOR=$(readlink -f "$1") - echo "File to monitor: $LOGGLY_FILE_TO_MONITOR" + LOGGLY_FILE_TO_MONITOR="$(readlink -f ${1%/})" + + if [ -f "$LOGGLY_FILE_TO_MONITOR" ];then + + LOGGLY_FILE_TO_MONITOR=$(readlink -f "$1") + FILE_TO_MONITOR=$LOGGLY_FILE_TO_MONITOR + echo "File to monitor: $LOGGLY_FILE_TO_MONITOR" + + elif [ -d "$LOGGLY_FILE_TO_MONITOR" ] || checkIfWildcardExist ; then + IS_DIRECTORY="true" + echo "Directory to monitor: $LOGGLY_FILE_TO_MONITOR" + + else + echo "ERROR: Cannot access $LOGGLY_FILE_TO_MONITOR: No such file or directory" + exit 1 + fi ;; -l | --filealias ) shift LOGGLY_FILE_TO_MONITOR_ALIAS=$1 + #keeping a copy of it as we need it in the loop + FILE_ALIAS=$LOGGLY_FILE_TO_MONITOR_ALIAS + STATE_FILE_ALIAS=$LOGGLY_FILE_TO_MONITOR_ALIAS CONF_FILE_FORMAT_NAME=$CONF_FILE_FORMAT_NAME$1 echo "File alias: $LOGGLY_FILE_TO_MONITOR_ALIAS" ;; @@ -373,6 +611,9 @@ if [ "$1" != "being-invoked" ]; then LOGGLY_FILE_TAG=$1 echo "File tag: $LOGGLY_FILE_TAG" ;; + -s | --suppress ) + SUPPRESS_PROMPT="true" + ;; -h | --help) usage exit diff --git a/Modular Scripts/Nginx/configure-nginx.sh b/Modular Scripts/Nginx/configure-nginx.sh index 3df7157..7ed1589 100644 --- a/Modular Scripts/Nginx/configure-nginx.sh +++ b/Modular Scripts/Nginx/configure-nginx.sh @@ -9,7 +9,7 @@ source configure-linux.sh "being-invoked" #name of the current script SCRIPT_NAME=configure-nginx.sh #version of the current script -SCRIPT_VERSION=1.0 +SCRIPT_VERSION=1.3 #we have not found the nginx version yet at this point in the script APP_TAG="\"nginx-version\":\"\"" @@ -32,10 +32,17 @@ LOGGLY_NGINX_LOG_HOME= #this variable will hold the users nginx version NGINX_VERSION= -MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure nginx is available at https://www.loggly.com/docs/nginx-server-logs#manual" +MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure nginx is available at https://www.loggly.com/docs/nginx-server-logs#manual. Rsyslog troubleshooting instructions are available at https://www.loggly.com/docs/troubleshooting-rsyslog/" #this variable will hold if the check env function for linux is invoked NGINX_ENV_VALIDATED="false" + +#apache as tag sent with the logs +LOGGLY_FILE_TAG="nginx" + +#add tags to the logs +TAG= + ########## Variable Declarations - End ########## #check if nginx environment is compatible for Loggly @@ -65,6 +72,9 @@ installLogglyConfForNginx() #configure loggly for Linux installLogglyConf + #multiple tags + addTagsInConfiguration + #create 21nginx.conf file write21NginxConfFile @@ -73,9 +83,9 @@ installLogglyConfForNginx() #verify if the nginx logs made it to loggly checkIfNginxLogsMadeToLoggly - + #log success message - logMsgToConfigSysLog "SUCCESS" "SUCCESS: Nginx successfully configured to send logs via Loggly." + logMsgToConfigSysLog "SUCCESS" "SUCCESS: Nginx successfully configured to send logs to Loggly." } #executing script to remove loggly configuration for Nginx @@ -98,6 +108,7 @@ removeLogglyConfForNginx() logMsgToConfigSysLog "INFO" "INFO: Rollback completed." } + #identify if nginx is installed on your system and is available as a service checkNginxDetails() { @@ -136,20 +147,24 @@ checkLogFileSize() errorFileSize=$(wc -c "$2" | cut -f 1 -d ' ') fileSize=$((accessFileSize+errorFileSize)) if [ $fileSize -ge 102400000 ]; then - logMsgToConfigSysLog "INFO" "INFO: " - while true; do - read -p "WARN: There are currently large log files which may use up your allowed volume. Please rotate your logs before continuing. Would you like to continue now anyway? (yes/no)" yn - case $yn in - [Yy]* ) - logMsgToConfigSysLog "INFO" "INFO: Current nginx logs size is $fileSize bytes. Continuing with nginx Loggly configuration."; - break;; - [Nn]* ) - logMsgToConfigSysLog "INFO" "INFO: Current nginx logs size is $fileSize bytes. Discontinuing with nginx Loggly configuration." - exit 1 - break;; - * ) echo "Please answer yes or no.";; - esac - done + if [ "$SUPPRESS_PROMPT" == "false" ]; then + while true; do + read -p "WARN: There are currently large log files which may use up your allowed volume. Please rotate your logs before continuing. Would you like to continue now anyway? (yes/no)" yn + case $yn in + [Yy]* ) + logMsgToConfigSysLog "INFO" "INFO: Current nginx logs size is $fileSize bytes. Continuing with nginx Loggly configuration."; + break;; + [Nn]* ) + logMsgToConfigSysLog "INFO" "INFO: Current nginx logs size is $fileSize bytes. Discontinuing with nginx Loggly configuration." + exit 1 + break;; + * ) echo "Please answer yes or no.";; + esac + done + else + logMsgToConfigSysLog "WARN" "WARN: There are currently large log files which may use up your allowed volume." + logMsgToConfigSysLog "INFO" "INFO: Current nginx logs size is $fileSize bytes. Continuing with nginx Loggly configuration."; + fi elif [ $fileSize -eq 0 ]; then logMsgToConfigSysLog "WARN" "WARN: There are no recent logs from nginx there so won't be any sent to Loggly. You can generate some logs by visiting a page on your web server." exit 1 @@ -161,24 +176,39 @@ write21NginxConfFile() #Create nginx syslog config file if it doesn't exist echo "INFO: Checking if nginx sysconf file $NGINX_SYSLOG_CONFFILE exist." if [ -f "$NGINX_SYSLOG_CONFFILE" ]; then - logMsgToConfigSysLog "WARN" "WARN: nginx syslog file $NGINX_SYSLOG_CONFFILE already exist." - while true; do - read -p "Do you wish to override $NGINX_SYSLOG_CONFFILE? (yes/no)" yn - case $yn in - [Yy]* ) - logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $NGINX_SYSLOG_CONFFILE to $NGINX_SYSLOG_CONFFILE_BACKUP"; - sudo mv -f $NGINX_SYSLOG_CONFFILE $NGINX_SYSLOG_CONFFILE_BACKUP; - write21NginxFileContents; - break;; - [Nn]* ) break;; - * ) echo "Please answer yes or no.";; - esac - done + logMsgToConfigSysLog "WARN" "WARN: nginx syslog file $NGINX_SYSLOG_CONFFILE already exist." + if [ "$SUPPRESS_PROMPT" == "false" ]; then + while true; do + read -p "Do you wish to override $NGINX_SYSLOG_CONFFILE? (yes/no)" yn + case $yn in + [Yy]* ) + logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $NGINX_SYSLOG_CONFFILE to $NGINX_SYSLOG_CONFFILE_BACKUP"; + sudo mv -f $NGINX_SYSLOG_CONFFILE $NGINX_SYSLOG_CONFFILE_BACKUP; + write21NginxFileContents; + break;; + [Nn]* ) break;; + * ) echo "Please answer yes or no.";; + esac + done + else + logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $NGINX_SYSLOG_CONFFILE to $NGINX_SYSLOG_CONFFILE_BACKUP"; + sudo mv -f $NGINX_SYSLOG_CONFFILE $NGINX_SYSLOG_CONFFILE_BACKUP; + write21NginxFileContents; + fi else write21NginxFileContents fi } +addTagsInConfiguration() +{ + #split tags by comman(,) + IFS=, read -a array <<< "$LOGGLY_FILE_TAG" + for i in "${array[@]}" + do + TAG="$TAG tag=\\\"$i\\\" " + done +} #function to write the contents of nginx syslog config file write21NginxFileContents() { @@ -213,7 +243,7 @@ write21NginxFileContents() \$InputRunFileMonitor #Add a tag for nginx events - \$template LogglyFormatNginx,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 tag=\\\"nginx\\\"] %msg%\n\" + \$template LogglyFormatNginx,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 $TAG] %msg%\n\" if \$programname == 'nginx-access' then @@logs-01.loggly.com:514;LogglyFormatNginx if \$programname == 'nginx-access' then ~ @@ -229,7 +259,6 @@ EOIPFW restartRsyslog } - #checks if the nginx logs made to loggly checkIfNginxLogsMadeToLoggly() { @@ -238,8 +267,19 @@ checkIfNginxLogsMadeToLoggly() nginxInitialLogCount=0 nginxLatestLogCount=0 - queryParam="tag%3Anginx&from=-15m&until=now&size=1" - + + TAGS= + IFS=, read -a array <<< "$LOGGLY_FILE_TAG" + for i in "${array[@]}" + do + if [ "$TAGS" == "" ]; then + TAGS="tag%3A$i" + else + TAGS="$TAGS%20tag%3A$i" + fi + done + + queryParam="$TAGS&from=-15m&until=now&size=1" queryUrl="$LOGGLY_ACCOUNT_URL/apiv2/search?q=$queryParam" logMsgToConfigSysLog "INFO" "INFO: Search URL: $queryUrl" @@ -267,8 +307,29 @@ checkIfNginxLogsMadeToLoggly() done if [ "$nginxLatestLogCount" -gt "$nginxInitialLogCount" ]; then - logMsgToConfigSysLog "SUCCESS" "SUCCESS: nginx logs successfully transferred to Loggly! You are now sending nginx logs to Loggly." - exit 0 + logMsgToConfigSysLog "INFO" "INFO: Nginx logs successfully transferred to Loggly! You are now sending Nginx logs to Loggly." + checkIfLogsAreParsedInLoggly + fi +} + +#verifying if the logs are being parsed or not +checkIfLogsAreParsedInLoggly() +{ + nginxInitialLogCount=0 + TAG_PARSER= + IFS=, read -a array <<< "$LOGGLY_FILE_TAG" + for i in "${array[@]}" + do + TAG_PARSER="$TAG_PARSER%20tag%3A$i " + done + queryParam="logtype%3Anginx$TAG_PARSER&from=-15m&until=now&size=1" + queryUrl="$LOGGLY_ACCOUNT_URL/apiv2/search?q=$queryParam" + searchAndFetch nginxInitialLogCount "$queryUrl" + logMsgToConfigSysLog "INFO" "INFO: Verifying if the Nginx logs are parsed in Loggly." + if [ "$nginxInitialLogCount" -gt 0 ]; then + logMsgToConfigSysLog "INFO" "INFO: Nginx logs successfully parsed in Loggly!" + else + logMsgToConfigSysLog "WARN" "WARN: We received your logs but they do not appear to use one of our automatically parsed formats. You can still do full text search and counts on these logs, but you won't be able to use our field explorer. Please consider switching to one of our automated formats https://www.loggly.com/docs/automated-parsing/" fi } @@ -287,7 +348,7 @@ remove21NginxConfFile() usage() { cat << EOF -usage: configure-nginx [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] +usage: configure-nginx [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-tag filetag1,filetag2 (optional)] [-s suppress prompts {optional)] usage: configure-nginx [-a loggly auth account or subdomain] [-r to rollback] usage: configure-nginx [-h for help] EOF @@ -316,9 +377,16 @@ while [ "$1" != "" ]; do -p | --password ) shift LOGGLY_PASSWORD=$1 ;; + -tag| --filetag ) shift + LOGGLY_FILE_TAG=$1 + echo "File tag: $LOGGLY_FILE_TAG" + ;; -r | --rollback ) LOGGLY_ROLLBACK="true" ;; + -s | --suppress ) + SUPPRESS_PROMPT="true" + ;; -h | --help) usage exit diff --git a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh index f14d841..c2c57f3 100644 --- a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh +++ b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh @@ -8,13 +8,21 @@ source configure-file-monitoring.sh "being-invoked" #name of the current script SCRIPT_NAME=configure-s3-file-monitoring.sh #version of the current script -SCRIPT_VERSION=1.3 +SCRIPT_VERSION=1.5 #s3 bucket name to configure LOGGLY_S3_BUCKET_NAME= #alias name, will be used as tag & state file name etc. provided by user LOGGLY_S3_ALIAS= +FILE_ALIAS= +STATE_FILE_ALIAS= + +#file as tag sent with the logs +LOGGLY_FILE_TAG="s3file" + +#format name for the conf file. Can be set by calling script +CONF_FILE_FORMAT_NAME="LogglyFormatS3" #file alias provided by the user APP_TAG="\"s3file-alias\":\"\"" @@ -33,7 +41,7 @@ TEMP_DIR= IS_S3CMD_CONFIGURED_BY_SCRIPT="false" -MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure a file is available at https://www.loggly.com/docs/file-monitoring/" +MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure a file is available at https://www.loggly.com/docs/file-monitoring/. Rsyslog troubleshooting instructions are available at https://www.loggly.com/docs/troubleshooting-rsyslog/" ########## Variable Declarations - End ########## @@ -55,9 +63,6 @@ installLogglyConfForS3() #check if s3bucket is valid checkIfValidS3Bucket - #configure loggly for Linux - installLogglyConf - #create temporary directory createTempDir @@ -211,18 +216,20 @@ createTempDir() if [ -d "$TEMP_DIR" ]; then if [ "$(ls -A $TEMP_DIR)" ]; then logMsgToConfigSysLog "WARN" "WARN: There are some files/folders already present in $TEMP_DIR. If you continue, the files currently inside the $TEMP_DIR will also be configured to send logs to loggly." - while true; do - read -p "Would you like to continue now anyway? (yes/no)" yn - case $yn in - [Yy]* ) - break;; - [Nn]* ) - logMsgToConfigSysLog "INFO" "INFO: Discontinuing with s3 file monitoring configuration." - exit 1 - break;; - * ) echo "Please answer yes or no.";; - esac - done + if [ "$SUPPRESS_PROMPT" == "false" ]; then + while true; do + read -p "Would you like to continue now anyway? (yes/no)" yn + case $yn in + [Yy]* ) + break;; + [Nn]* ) + logMsgToConfigSysLog "INFO" "INFO: Discontinuing with s3 file monitoring configuration." + exit 1 + break;; + * ) echo "Please answer yes or no.";; + esac + done + fi fi else if [ -d "/tmp/s3monitoring" ]; then @@ -248,37 +255,15 @@ downloadS3Bucket() fi } - invokeS3FileMonitoring() { dir=/tmp/s3monitoring/$LOGGLY_S3_ALIAS - #TODO: Not supporting multiple files with same name in different directories - #only supporting file with naming convention *.* - for f in $(find $dir -name '*') - do - fileNameWithExt=${f##*/} - uniqueFileName=$(echo "$fileNameWithExt" | tr . _) - var=$(file $f) - - #it may be possible that the "text" may contain some uppercase letters like "Text" - var=$(echo $var | tr "[:upper:]" "[:lower:]") - - if [[ $var == *text* ]]; then - LOGGLY_FILE_TO_MONITOR_ALIAS=$uniqueFileName-$LOGGLY_S3_ALIAS - LOGGLY_FILE_TO_MONITOR=$f - LOGGLY_FILE_TAG="s3file" - CONF_FILE_FORMAT_NAME="LogglyFormatS3" - constructFileVariables - checkFileReadPermission - checkLogFileSize $LOGGLY_FILE_TO_MONITOR - write21ConfFileContents - IS_ANY_FILE_CONFIGURED="true" - #ignoring directory - elif [[ $var != *directory* ]]; then - logMsgToConfigSysLog "WARN" "WARN: File $fileNameWithExt is not a text file. Ignoring." - fi - done + LOGGLY_FILE_TO_MONITOR=$dir + configureDirectoryFileMonitoring + + IS_ANY_FILE_CONFIGURED="true" + if [ "$IS_ANY_FILE_CONFIGURED" != "false" ]; then restartRsyslog fi @@ -286,64 +271,71 @@ invokeS3FileMonitoring() installCronToSyncS3BucketPeriodically() { - while true; do - read -p "Would you like install a Cron job to sync the files currently in your bucket every 5 minutes? (yes/no)" yn - case $yn in - [Yy]* ) - - #copying .s3cfg file to /root so that it can be used by crontab for sync - if ! sudo test -f "/root/.s3cfg" ; then - sudo cp $HOME/.s3cfg /root - fi - - CRON_FILE="/tmp/s3monitoring/cron_$LOGGLY_S3_ALIAS" - CRON_SYNC_PATH="/tmp/s3monitoring/$LOGGLY_S3_ALIAS" - - #checking if the provided s3 path if of directory or file - IS_DIR="true" - BUCKET_URL_LAST_VALUE=$(echo ${LOGGLY_S3_BUCKET_NAME##*/}) - - if [ "$BUCKET_URL_LAST_VALUE" != "" ]; then - for fle in $(find $CRON_SYNC_PATH -name $BUCKET_URL_LAST_VALUE) - do - if [ -f $fle ]; then - IS_DIR="false" - break - fi - done - fi - - #adding file name to the sync folder as the bucket path is - #provided upto a file - if [ "$IS_DIR" == "false" ]; then - CRON_SYNC_PATH="$CRON_SYNC_PATH/$BUCKET_URL_LAST_VALUE" - fi - - logMsgToConfigSysLog "INFO" "INFO: Creating a Cron job to sync $LOGGLY_S3_BUCKET_NAME files to $CRON_SYNC_PATH in every five minutes." - - #setting up cron job - CRON_JOB_TO_SYNC_S3_BUCKET="*/5 * * * * s3cmd sync $LOGGLY_S3_BUCKET_NAME --preserve $CRON_SYNC_PATH" - - EXISTING_CRONS=$(sudo crontab -l 2>&1) - case $EXISTING_CRONS in - no*) - ;; - *) - echo "$EXISTING_CRONS" >> $CRON_FILE - ;; - esac - - echo "$CRON_JOB_TO_SYNC_S3_BUCKET" >> $CRON_FILE - sudo crontab $CRON_FILE - sudo rm -fr $CRON_FILE - break;; - [Nn]* ) - logMsgToConfigSysLog "INFO" "INFO: Skipping Cron installation." - break;; - * ) echo "Please answer yes or no.";; - esac - done + if [ "$SUPPRESS_PROMPT" == "false" ]; then + while true; do + read -p "Would you like install a Cron job to sync the files currently in your bucket every 5 minutes? (yes/no)" yn + case $yn in + [Yy]* ) + doS3CronInstallation + break;; + [Nn]* ) + logMsgToConfigSysLog "INFO" "INFO: Skipping Cron installation." + break;; + * ) echo "Please answer yes or no.";; + esac + done + else + doS3CronInstallation + fi +} + +doS3CronInstallation() +{ + #copying .s3cfg file to /root so that it can be used by crontab for sync + if ! sudo test -f "/root/.s3cfg" ; then + sudo cp $HOME/.s3cfg /root + fi + + CRON_FILE="/tmp/s3monitoring/cron_$LOGGLY_S3_ALIAS" + CRON_SYNC_PATH="/tmp/s3monitoring/$LOGGLY_S3_ALIAS" + + #checking if the provided s3 path if of directory or file + IS_DIR="true" + BUCKET_URL_LAST_VALUE=$(echo ${LOGGLY_S3_BUCKET_NAME##*/}) + + if [ "$BUCKET_URL_LAST_VALUE" != "" ]; then + for fle in $(find $CRON_SYNC_PATH -name $BUCKET_URL_LAST_VALUE) + do + if [ -f $fle ]; then + IS_DIR="false" + break + fi + done + fi + + #adding file name to the sync folder as the bucket path is + #provided upto a file + if [ "$IS_DIR" == "false" ]; then + CRON_SYNC_PATH="$CRON_SYNC_PATH/$BUCKET_URL_LAST_VALUE" + fi + + logMsgToConfigSysLog "INFO" "INFO: Creating a Cron job to sync $LOGGLY_S3_BUCKET_NAME files to $CRON_SYNC_PATH in every five minutes." + + #setting up cron job + CRON_JOB_TO_SYNC_S3_BUCKET="*/5 * * * * s3cmd sync $LOGGLY_S3_BUCKET_NAME --preserve $CRON_SYNC_PATH" + + EXISTING_CRONS=$(sudo crontab -l 2>&1) + case $EXISTING_CRONS in + no*) + ;; + *) + echo "$EXISTING_CRONS" >> $CRON_FILE + ;; + esac + echo "$CRON_JOB_TO_SYNC_S3_BUCKET" >> $CRON_FILE + sudo crontab $CRON_FILE + sudo rm -fr $CRON_FILE } deleteTempDir() @@ -429,7 +421,7 @@ deleteS3CronFromCrontab() usage() { cat << EOF -usage: configure-s3-file-monitoring [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-s3url s3bucketname ] [-s3l s3alias] +usage: configure-s3-file-monitoring [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-s3url s3bucketname ] [-s3l s3alias] [-s suppress prompts {optional)] usage: configure-s3-file-monitoring [-a loggly auth account or subdomain] [-r to rollback] [-s3l s3alias] usage: configure-s3-file-monitoring [-h for help] EOF @@ -468,8 +460,13 @@ while [ "$1" != "" ]; do -s3l | --s3alias ) shift LOGGLY_S3_ALIAS=$1 - echo "File alias: $LOGGLY_S3_ALIAS" + FILE_ALIAS=$LOGGLY_S3_ALIAS + STATE_FILE_ALIAS=$LOGGLY_S3_ALIAS + echo "File alias: $FILE_ALIAS" ;; + -s | --suppress ) + SUPPRESS_PROMPT="true" + ;; -h | --help) usage exit diff --git a/Modular Scripts/Tomcat/configure-tomcat.sh b/Modular Scripts/Tomcat/configure-tomcat.sh index 91f92ea..6a93d0d 100644 --- a/Modular Scripts/Tomcat/configure-tomcat.sh +++ b/Modular Scripts/Tomcat/configure-tomcat.sh @@ -9,7 +9,7 @@ source configure-linux.sh "being-invoked" #name of the current script SCRIPT_NAME=configure-tomcat.sh #version of the current script -SCRIPT_VERSION=1.1 +SCRIPT_VERSION=1.3 #minimum version of tomcat to enable log rotation MIN_TOMCAT_VERSION=6.0.33.0 @@ -43,11 +43,17 @@ TOMCAT_VERSION= #this variable will hold the location of log4j files path LOG4J_FILE_PATH= +#tomcat as tag sent with the logs +LOGGLY_FILE_TAG="tomcat" + +#add tags to the logs +TAG= + #this variable will hold the catalina home provide by user. #this is not a mandatory input LOGGLY_CATALINA_HOME= -MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure Tomcat is available at https://www.loggly.com/docs/tomcat-application-server" +MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure Tomcat is available at https://www.loggly.com/docs/tomcat-application-server/. Rsyslog troubleshooting instructions are available at https://www.loggly.com/docs/troubleshooting-rsyslog/" #this variable will hold if the check env function for linux is invoked TOMCAT_ENV_VALIDATED= @@ -91,7 +97,10 @@ installLogglyConfForTomcat() #update logging.properties file for log rotation updateLoggingPropertiesFile - + + #multiple tags + addTagsInConfiguration + #create 21tomcat.conf file write21TomcatConfFile @@ -334,18 +343,22 @@ checkIfTomcatConfiguredWithLog4J() canTomcatBeRestarted() { - while true; do - read -p "Tomcat needs to be restarted during configuration. Do you wish to continue? (yes/no)" yn - case $yn in - [Yy]* ) - break;; - [Nn]* ) - logMsgToConfigSysLog "WARN" "WARN: This script must restart Tomcat. Please run the script again when you are ready to restart it. No changes have been made to your system. Exiting." - exit 1 - break;; - * ) echo "Please answer yes or no.";; - esac - done + if [ "$SUPPRESS_PROMPT" == "false" ]; then + while true; do + read -p "Tomcat needs to be restarted during configuration. Do you wish to continue? (yes/no)" yn + case $yn in + [Yy]* ) + break;; + [Nn]* ) + logMsgToConfigSysLog "WARN" "WARN: This script must restart Tomcat. Please run the script again when you are ready to restart it. No changes have been made to your system. Exiting." + exit 1 + break;; + * ) echo "Please answer yes or no.";; + esac + done + else + logMsgToConfigSysLog "WARN" "WARN:Tomcat needs to be restarted during configuration." + fi } #backup the logging.properties file in the CATALINA_HOME folder backupLoggingPropertiesFile() @@ -414,24 +427,41 @@ EOIPFW fi } + +addTagsInConfiguration() +{ + #split tags by comman(,) + IFS=, read -a array <<< "$LOGGLY_FILE_TAG" + for i in "${array[@]}" + do + TAG="$TAG tag=\\\"$i\\\" " + done +} + write21TomcatConfFile() { #Create tomcat syslog config file if it doesn't exist echo "INFO: Checking if tomcat sysconf file $TOMCAT_SYSLOG_CONFFILE exist." if [ -f "$TOMCAT_SYSLOG_CONFFILE" ]; then logMsgToConfigSysLog "WARN" "WARN: Tomcat syslog file $TOMCAT_SYSLOG_CONFFILE already exist." - while true; do - read -p "Do you wish to override $TOMCAT_SYSLOG_CONFFILE? (yes/no)" yn - case $yn in - [Yy]* ) - logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $TOMCAT_SYSLOG_CONFFILE to $TOMCAT_SYSLOG_CONFFILE_BACKUP"; - sudo mv -f $TOMCAT_SYSLOG_CONFFILE $TOMCAT_SYSLOG_CONFFILE_BACKUP; - write21TomcatFileContents; - break;; - [Nn]* ) break;; - * ) echo "Please answer yes or no.";; - esac - done + if [ "$SUPPRESS_PROMPT" == "false" ]; then + while true; do + read -p "Do you wish to override $TOMCAT_SYSLOG_CONFFILE? (yes/no)" yn + case $yn in + [Yy]* ) + logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $TOMCAT_SYSLOG_CONFFILE to $TOMCAT_SYSLOG_CONFFILE_BACKUP"; + sudo mv -f $TOMCAT_SYSLOG_CONFFILE $TOMCAT_SYSLOG_CONFFILE_BACKUP; + write21TomcatFileContents; + break;; + [Nn]* ) break;; + * ) echo "Please answer yes or no.";; + esac + done + else + logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $TOMCAT_SYSLOG_CONFFILE to $TOMCAT_SYSLOG_CONFFILE_BACKUP"; + sudo mv -f $TOMCAT_SYSLOG_CONFFILE $TOMCAT_SYSLOG_CONFFILE_BACKUP; + write21TomcatFileContents; + fi else write21TomcatFileContents fi @@ -455,7 +485,7 @@ write21TomcatFileContents() imfileStr+=" #parameterized token here....... #Add a tag for tomcat events - \$template LogglyFormatTomcat,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 tag=\\\"tomcat\\\"] %msg%\n\" + \$template LogglyFormatTomcat,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 $TAG] %msg%\n\" # catalina.out \$InputFileName $LOGGLY_CATALINA_LOG_HOME/catalina.out @@ -643,7 +673,7 @@ restartTomcat() usage() { cat << EOF -usage: configure-tomcat [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-ch catalina home (optional)] +usage: configure-tomcat [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-ch catalina home (optional)] [-tag filetag1,filetag2 (optional)] [-s suppress prompts {optional)] usage: configure-tomcat [-r to rollback] [-a loggly auth account or subdomain] [-ch catalina home (optional)] usage: configure-tomcat [-h for help] EOF @@ -675,10 +705,17 @@ while [ "$1" != "" ]; do ;; -p | --password ) shift LOGGLY_PASSWORD=$1 - ;; + ;; + -tag| --filetag ) shift + LOGGLY_FILE_TAG=$1 + echo "File tag: $LOGGLY_FILE_TAG" + ;; -r | --rollback ) - LOGGLY_ROLLBACK="true" + LOGGLY_ROLLBACK="true" ;; + -s | --suppress ) + SUPPRESS_PROMPT="true" + ;; -h | --help) usage exit diff --git a/publish.sh b/publish.sh new file mode 100644 index 0000000..3289a08 --- /dev/null +++ b/publish.sh @@ -0,0 +1,10 @@ +# Publishes the scripts to Loggly's S3 bucket where they are publically hosted. +# For Loggly's internal use only. Requires keys to publish. + +s3cmd put --acl-public Linux\ Script/configure-linux.sh s3://loggly-install/install/ +s3cmd put --acl-public Modular\ Scripts/File\ Monitoring/configure-file-monitoring.sh s3://loggly-install/install/ +s3cmd put --acl-public Modular\ Scripts/Apache2/configure-apache.sh s3://loggly-install/install/ +s3cmd put --acl-public Modular\ Scripts/Nginx/configure-nginx.sh s3://loggly-install/install/ +s3cmd put --acl-public Modular\ Scripts/S3Logs\ Monitoring/configure-s3-file-monitoring.sh s3://loggly-install/install/ +s3cmd put --acl-public Modular\ Scripts/Tomcat/configure-tomcat.sh s3://loggly-install/install/ +s3cmd put --acl-public Mac\ Script/configure-mac.sh s3://loggly-install/install/