From 513e512ed87fb21c55096569c574532970d0c506 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= <markus.makela@mariadb.com>
Date: Sun, 11 Feb 2024 14:05:35 +0200
Subject: [PATCH] Add trace logging

The trace log writes all log events to a set of rotating files. These
files are intended to be stored on a tmpfs mount where they can be used
to keep a rolling log that can then be used to analyze problems
after-the-fact.
---
 include/maxscale/config.hh |   5 ++
 include/maxscale/log.hh    |   7 +++
 server/core/config.cc      |  25 ++++++--
 server/core/gateway.cc     |   1 +
 server/core/log.cc         | 115 +++++++++++++++++++++++++++++++++++++
 5 files changed, 149 insertions(+), 4 deletions(-)

diff --git a/include/maxscale/config.hh b/include/maxscale/config.hh
index d59559359f..73ae5bb0f2 100644
--- a/include/maxscale/config.hh
+++ b/include/maxscale/config.hh
@@ -183,6 +183,9 @@ public:
     config::Count        rebalance_window;          /**< How many seconds should be taken into account. */
     config::Bool         skip_name_resolve;         /**< Reverse DNS lookups */
 
+    config::Path trace_file_dir;
+    config::Size trace_file_size;
+
     // NON-modifiable automatically configured parameters.
     int64_t       n_threads;                    /**< Number of polling threads */
     std::string   qc_name;                      /**< The name of the query classifier to load */
@@ -285,6 +288,8 @@ private:
     static config::ParamMilliseconds                    s_rebalance_period;
     static config::ParamCount                           s_rebalance_window;
     static config::ParamBool                            s_skip_name_resolve;
+    static config::ParamPath                            s_trace_file_dir;
+    static config::ParamSize                            s_trace_file_size;
 
     static ParamThreadsCount                            s_n_threads;
     static config::ParamString                          s_qc_name;
diff --git a/include/maxscale/log.hh b/include/maxscale/log.hh
index f964fea027..bca3b436a2 100644
--- a/include/maxscale/log.hh
+++ b/include/maxscale/log.hh
@@ -111,6 +111,13 @@ inline void mxs_log_finish()
     mxb_log_finish();
 }
 
+/**
+ * Initializes the trace log file
+ *
+ * This should be called after the global configuration has been read and must not be called again after that.
+ */
+void mxs_init_trace_file();
+
 #define MXS_LOG_MESSAGE MXB_LOG_MESSAGE
 
 #define MXS_ALERT   MXB_ALERT
diff --git a/server/core/config.cc b/server/core/config.cc
index 93b1db502c..3cfbf93b72 100644
--- a/server/core/config.cc
+++ b/server/core/config.cc
@@ -484,6 +484,18 @@ config::ParamBool Config::s_skip_name_resolve(
     false,
     config::Param::Modifiable::AT_RUNTIME);
 
+config::ParamPath Config::s_trace_file_dir(
+    &Config::s_specification, "trace_file_dir",
+    "Directory where trace files are stored at",
+    config::ParamPath::C | config::ParamPath::W,
+    "");
+
+config::ParamSize Config::s_trace_file_size(
+    &Config::s_specification, "trace_file_size",
+    "How many bytes of the trace log is retained",
+    0,
+    config::Param::AT_RUNTIME);
+
 Config::ParamThreadsCount Config::s_n_threads(
     &Config::s_specification,
     CN_THREADS,
@@ -781,10 +793,7 @@ Config::Config(int argc, char** argv)
     dump_statements(this, &s_dump_statements, [](session_dump_statements_t when) {
                         session_set_dump_statements(when);
                     }),
-    session_trace(this, &s_session_trace, [](int32_t count) {
-                      session_set_session_trace(count);
-                      mxb_log_set_session_trace(count > 0 ? true : false);
-                  }),
+    session_trace(this, &s_session_trace),
     session_trace_match(this, &s_session_trace_match),
     ms_timestamp(this, &s_ms_timestamp, [](bool enable) {
                      mxb_log_set_highprecision_enabled(enable);
@@ -828,6 +837,8 @@ Config::Config(int argc, char** argv)
                      }),
     rebalance_window(this, &s_rebalance_window),
     skip_name_resolve(this, &s_skip_name_resolve),
+    trace_file_dir(this, &s_trace_file_dir),
+    trace_file_size(this, &s_trace_file_size),
     config_check(false),
     log_target(MXB_LOG_TARGET_DEFAULT),
     substitute_variables(false),
@@ -1046,6 +1057,12 @@ bool Config::post_configure(const std::map<std::string, mxs::ConfigParameters>&
         }
     }
 
+    int count = this->session_trace.get();
+    int trace_sz = this->trace_file_size.get();
+
+    session_set_session_trace(count);
+    mxb_log_set_session_trace(count || trace_sz);
+
     return rv;
 }
 
diff --git a/server/core/gateway.cc b/server/core/gateway.cc
index 1e95d3c949..124bb0ef6e 100644
--- a/server/core/gateway.cc
+++ b/server/core/gateway.cc
@@ -2226,6 +2226,7 @@ int main(int argc, char** argv)
                 // we need to exit. After this point, the shutdown will be driven by the workers.
                 if (!maxscale_is_shutting_down())
                 {
+                    mxs_init_trace_file();
 
                     // Start the routing workers, each in a thread of its own.
                     if (RoutingWorker::start_workers())
diff --git a/server/core/log.cc b/server/core/log.cc
index 6703051367..0d4459a544 100644
--- a/server/core/log.cc
+++ b/server/core/log.cc
@@ -16,10 +16,12 @@
 
 #include <sys/time.h>
 #include <syslog.h>
+#include <fcntl.h>
 
 #include <atomic>
 #include <cinttypes>
 #include <fstream>
+#include <iostream>
 
 #ifdef HAVE_SYSTEMD
 #include <systemd/sd-journal.h>
@@ -39,15 +41,111 @@
 
 namespace
 {
+const int64_t MAX_TRACE_FILE_COUNT = 10;
 
 struct ThisUnit
 {
     std::atomic<int> rotation_count {0};
+
+    int        trace_fd {-1};
+    std::mutex trace_lock;
+    int64_t    trace_file_num {0};
+
+    ~ThisUnit()
+    {
+        if (this->trace_fd != -1)
+        {
+            close(this->trace_fd);
+        }
+    }
 };
 ThisUnit this_unit;
 
 const char* LOGFILE_NAME = "maxscale.log";
 
+bool file_too_big(int64_t trace_file_size)
+{
+    return lseek(this_unit.trace_fd, 0, SEEK_CUR) >= (trace_file_size / MAX_TRACE_FILE_COUNT);
+}
+
+std::string trace_file_name(int seq)
+{
+    // The PID of the process is not included in the file name. This avoids stale files if MaxScale is
+    // restarted while the tracing is enabled.
+    return MAKE_STR(mxs::Config::get().trace_file_dir.get() << "/maxscale.trace." << seq);
+}
+
+int open_trace_file(const std::string& filename)
+{
+    int fd = open(filename.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_APPEND, 0622);
+
+    if (fd == -1)
+    {
+        std::cout << "Failed to open trace log file '" << filename << "': "
+                  << errno << ", " << mxb_strerror(errno) << "\n";
+    }
+
+    return fd;
+}
+
+void rotate_trace_file()
+{
+    auto next_name = trace_file_name(this_unit.trace_file_num++);
+    int new_fd = open_trace_file(next_name);
+
+    if (new_fd != -1)
+    {
+        dup2(new_fd, this_unit.trace_fd);
+        close(new_fd);
+
+        if (this_unit.trace_file_num > MAX_TRACE_FILE_COUNT)
+        {
+            // Remove trace file 0
+            std::string prev = trace_file_name(0);
+            unlink(prev.c_str());
+
+            // Rotate file names from trace N to trace N-1
+            for (int i = 1; i < MAX_TRACE_FILE_COUNT; i++)
+            {
+                std::string current = trace_file_name(i);
+                rename(current.c_str(), prev.c_str());
+                prev = std::move(current);
+            }
+
+            // Rename the newly created file as the last trace file
+            rename(next_name.c_str(), prev.c_str());
+        }
+    }
+}
+
+void write_to_trace(const std::string& msg, int64_t trace_file_size)
+{
+    if (this_unit.trace_fd == -1)
+    {
+        return;
+    }
+
+    int rc = write(this_unit.trace_fd, msg.data(), msg.size());
+
+    if (rc == -1)
+    {
+        std::cout << "Failed to write trace log: " << errno << ", " << mxb_strerror(errno) << "\n";
+    }
+    else if ((size_t)rc < msg.size())
+    {
+        std::cout << "Write to trace log was only partially complete.\n";
+    }
+    else if (file_too_big(trace_file_size))
+    {
+        std::unique_lock<std::mutex> guard(this_unit.trace_lock, std::try_to_lock);
+
+        if (guard && file_too_big(trace_file_size))
+        {
+            rotate_trace_file();
+        }
+    }
+}
+
 size_t mxs_get_context(char* buffer, size_t len)
 {
     mxb_assert(len >= 20);      // Needed for "9223372036854775807"
@@ -73,6 +171,13 @@ void mxs_log_in_memory(const std::string& msg)
     {
         session->append_session_log(msg);
     }
+
+    int64_t trace_file_size = mxs::Config::get().trace_file_size.atomic_get();
+
+    if (trace_file_size > 0)
+    {
+        write_to_trace(msg, trace_file_size);
+    }
 }
 
 bool mxs_should_log(int priority)
@@ -90,6 +195,16 @@ bool mxs_log_init(const char* ident, const char* logdir, mxs_log_target_t target
                         mxs_get_context, mxs_log_in_memory, mxs_should_log);
 }
 
+void mxs_init_trace_file()
+{
+    auto trace_dir = mxs::Config::get().trace_file_dir.get();
+
+    if (!trace_dir.empty())
+    {
+        this_unit.trace_fd = open_trace_file(trace_file_name(this_unit.trace_file_num++));
+    }
+}
+
 namespace
 {
 
-- 
2.43.0

