Include postgres logs into bubble's log management (#59)

Run log manager within bash in cron

Add logging into log manager script

Include postgres logs in log manager script

Link postgres log folder from withing bubble log folder

Redirect log_manager's stderr and stdout to log file

Add missing date in init bubble script log

Co-authored-by: Kristijan Mitrovic <kmitrovic@itekako.com>
Reviewed-on: #59

bubble-server/src/main/resources/packer/roles/bubble/files/init_bubble_db.sh

@@ -13,7 +13,7 @@ function die {
 }
 
 function log {
-  echo "${1}" >> ${LOG}
+  echo "$(date): ${1}" >> ${LOG}
 }
 
 export LANG="en_US.UTF-8"

bubble-server/src/main/resources/packer/roles/bubble/files/log_manager.sh

@@ -2,23 +2,52 @@
 #
 # Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
 #
+LOG=/var/log/bubble/log_manager.log
+
+function log {
+  echo "$(date): ${1}" >> ${LOG}
+}
+
 BUBBLE_LOGS_FOLDER=/var/log/bubble
+POSTGRES_LOGS_FOLDER=$(readlink -f "${BUBBLE_LOGS_FOLDER}"/postgresql)
 REDIS_LOG_FLAG_KEY="bubble.StandardSelfNodeService.bubble_server_logs_enabled"
 
 REDIS_LOG_FLAG_VALUE=$(echo "get ${REDIS_LOG_FLAG_KEY}" | redis-cli | xargs echo | tr '[:upper:]' '[:lower:]')
 
+log "starting log manager with REDIS_LOG_FLAG_VALUE=${REDIS_LOG_FLAG_VALUE}"
 if [[ ${REDIS_LOG_FLAG_VALUE} == true ]]; then
   is_reload_needed=false
-  for logFile in $(find "${BUBBLE_LOGS_FOLDER}"/* -type l); do
+  is_psql_restart_needed=false
+  # Cannot use -L option in find here as links are actually find's target:
+  for logFile in $(find "${BUBBLE_LOGS_FOLDER}"/* -type l ! -name postgresql); do
+    log "recreating real bubble log file: ${logFile}"
     rm "${logFile}"
     touch "${logFile}"
+    if [[ "${logFile}" == "${LOG}" ]]; then
+      log "...starting fresh log after activation..."
+    fi
     is_reload_needed=true
   done
+  for psqlLogFile in $(find "${POSTGRES_LOGS_FOLDER}"/* -type l); do
+    log "removing postgres link log file making room for a real one: ${logFile}"
+    rm "${psqlLogFile}"
+    is_psql_restart_needed=true
+  done
+
+  if [[ ${is_psql_restart_needed} == true ]]; then
+    log "restarting postgres service"
+    service postgresql restart
+  fi
   if [[ ${is_reload_needed} == true ]]; then
+    log "reloading supervisor"
     supervisorctl reload
   fi
 else
-  for logFile in $(find "${BUBBLE_LOGS_FOLDER}"/* -type f); do
+  # following dir link with -L option, so no need for special postgres for loop in this case:
+  for logFile in $(find -L "${BUBBLE_LOGS_FOLDER}"/* -type f); do
+    log "force-creating link to /dev/null instead of log file ${logFile}"
     ln -sf /dev/null "${logFile}"
   done
 fi
+
+log "ending log manager"

bubble-server/src/main/resources/packer/roles/bubble/tasks/main.yml

@@ -120,7 +120,7 @@
     name: "Log flag check and manager"
     minute: "*/5"
     user: "root"
-    job: "log_manager.sh"
+    job: "bash -c \"/usr/local/sbin/log_manager.sh >>/var/log/bubble/log_manager.log 2>&1\""
 
 - name: Install packer for sage node
   shell: su - bubble bash -c install_packer.sh

bubble-server/src/main/resources/packer/roles/bubble/tasks/postgresql.yml

@@ -36,3 +36,11 @@
     special_time: "hourly"
     user: "postgres"
     job: "HOUR_AGO=$(date -d '1 month ago' +\"%s000\") && psql -d bubble -c \"DELETE FROM sent_notification WHERE mtime < ${HOUR_AGO}\" -c \"DELETE FROM received_notification WHERE mtime < ${HOUR_AGO}\""
+
+- name: Link to postgres log dir from bubble's log dir
+  file:
+    src: /var/log/postgresql
+    dest: /var/log/bubble/postgresql
+    owner: root
+    group: postgres
+    state: link