From a39cb29480b0d83790dd1628a18c67c689c8253b Mon Sep 17 00:00:00 2001 From: Kristijan Mitrovic Date: Thu, 24 Sep 2020 14:49:21 +0000 Subject: [PATCH] Include postgres logs into bubble's log management (#59) Run log manager within bash in cron Add logging into log manager script Include postgres logs in log manager script Link postgres log folder from withing bubble log folder Redirect log_manager's stderr and stdout to log file Add missing date in init bubble script log Co-authored-by: Kristijan Mitrovic Reviewed-on: https://git.bubblev.org/bubblev/bubble/pulls/59 --- .../roles/bubble/files/init_bubble_db.sh | 2 +- .../packer/roles/bubble/files/log_manager.sh | 33 +++++++++++++++++-- .../packer/roles/bubble/tasks/main.yml | 2 +- .../packer/roles/bubble/tasks/postgresql.yml | 8 +++++ 4 files changed, 41 insertions(+), 4 deletions(-) diff --git a/bubble-server/src/main/resources/packer/roles/bubble/files/init_bubble_db.sh b/bubble-server/src/main/resources/packer/roles/bubble/files/init_bubble_db.sh index a70251ad..b79bf8dc 100644 --- a/bubble-server/src/main/resources/packer/roles/bubble/files/init_bubble_db.sh +++ b/bubble-server/src/main/resources/packer/roles/bubble/files/init_bubble_db.sh @@ -13,7 +13,7 @@ function die { } function log { - echo "${1}" >> ${LOG} + echo "$(date): ${1}" >> ${LOG} } export LANG="en_US.UTF-8" diff --git a/bubble-server/src/main/resources/packer/roles/bubble/files/log_manager.sh b/bubble-server/src/main/resources/packer/roles/bubble/files/log_manager.sh index 7003ed0f..2f18f7a8 100755 --- a/bubble-server/src/main/resources/packer/roles/bubble/files/log_manager.sh +++ b/bubble-server/src/main/resources/packer/roles/bubble/files/log_manager.sh @@ -2,23 +2,52 @@ # # Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ # +LOG=/var/log/bubble/log_manager.log + +function log { + echo "$(date): ${1}" >> ${LOG} +} + BUBBLE_LOGS_FOLDER=/var/log/bubble +POSTGRES_LOGS_FOLDER=$(readlink -f "${BUBBLE_LOGS_FOLDER}"/postgresql) REDIS_LOG_FLAG_KEY="bubble.StandardSelfNodeService.bubble_server_logs_enabled" REDIS_LOG_FLAG_VALUE=$(echo "get ${REDIS_LOG_FLAG_KEY}" | redis-cli | xargs echo | tr '[:upper:]' '[:lower:]') +log "starting log manager with REDIS_LOG_FLAG_VALUE=${REDIS_LOG_FLAG_VALUE}" if [[ ${REDIS_LOG_FLAG_VALUE} == true ]]; then is_reload_needed=false - for logFile in $(find "${BUBBLE_LOGS_FOLDER}"/* -type l); do + is_psql_restart_needed=false + # Cannot use -L option in find here as links are actually find's target: + for logFile in $(find "${BUBBLE_LOGS_FOLDER}"/* -type l ! -name postgresql); do + log "recreating real bubble log file: ${logFile}" rm "${logFile}" touch "${logFile}" + if [[ "${logFile}" == "${LOG}" ]]; then + log "...starting fresh log after activation..." + fi is_reload_needed=true done + for psqlLogFile in $(find "${POSTGRES_LOGS_FOLDER}"/* -type l); do + log "removing postgres link log file making room for a real one: ${logFile}" + rm "${psqlLogFile}" + is_psql_restart_needed=true + done + + if [[ ${is_psql_restart_needed} == true ]]; then + log "restarting postgres service" + service postgresql restart + fi if [[ ${is_reload_needed} == true ]]; then + log "reloading supervisor" supervisorctl reload fi else - for logFile in $(find "${BUBBLE_LOGS_FOLDER}"/* -type f); do + # following dir link with -L option, so no need for special postgres for loop in this case: + for logFile in $(find -L "${BUBBLE_LOGS_FOLDER}"/* -type f); do + log "force-creating link to /dev/null instead of log file ${logFile}" ln -sf /dev/null "${logFile}" done fi + +log "ending log manager" diff --git a/bubble-server/src/main/resources/packer/roles/bubble/tasks/main.yml b/bubble-server/src/main/resources/packer/roles/bubble/tasks/main.yml index 2a8235bb..e870a4dc 100644 --- a/bubble-server/src/main/resources/packer/roles/bubble/tasks/main.yml +++ b/bubble-server/src/main/resources/packer/roles/bubble/tasks/main.yml @@ -120,7 +120,7 @@ name: "Log flag check and manager" minute: "*/5" user: "root" - job: "log_manager.sh" + job: "bash -c \"/usr/local/sbin/log_manager.sh >>/var/log/bubble/log_manager.log 2>&1\"" - name: Install packer for sage node shell: su - bubble bash -c install_packer.sh diff --git a/bubble-server/src/main/resources/packer/roles/bubble/tasks/postgresql.yml b/bubble-server/src/main/resources/packer/roles/bubble/tasks/postgresql.yml index 19e33b4d..d2b8e486 100644 --- a/bubble-server/src/main/resources/packer/roles/bubble/tasks/postgresql.yml +++ b/bubble-server/src/main/resources/packer/roles/bubble/tasks/postgresql.yml @@ -36,3 +36,11 @@ special_time: "hourly" user: "postgres" job: "HOUR_AGO=$(date -d '1 month ago' +\"%s000\") && psql -d bubble -c \"DELETE FROM sent_notification WHERE mtime < ${HOUR_AGO}\" -c \"DELETE FROM received_notification WHERE mtime < ${HOUR_AGO}\"" + +- name: Link to postgres log dir from bubble's log dir + file: + src: /var/log/postgresql + dest: /var/log/bubble/postgresql + owner: root + group: postgres + state: link