Uploaded image for project: 'MariaDB Server'
  1. MariaDB Server
  2. MDEV-35694

Mysqlbinlog --stop-position should warn if EOF not reached with --read-from-remote-server

    XMLWordPrintable

Details

    Description

      MDEV-27037 added functionality to warn users that a specified stop-position or stop-datetime was never reached. It only worked for local files though. The patch in MDEV-35528 changed the implementation for stop-datetime to work to provide the warning also when using --read-from-remote-server. The PR for that MDEV (#3670) was limited to only the stop-datetime field.

      The stop-position field should also be updated to work with --read-from-remote-server. A patch was prototyped to add this functionality, which includes test fixes as well (patch content at the bottom of this description).

      Priority is critical, as MDEV-35528 leaves the code in mysqlbinlog in an inconsistent state, and we should try to address this before the release.

      diff --git a/client/mysqlbinlog.cc b/client/mysqlbinlog.cc
      index 2874b3e881b..c6c6325873e 100644
      --- a/client/mysqlbinlog.cc
      +++ b/client/mysqlbinlog.cc
      @@ -143,6 +143,7 @@ static uint verbose= 0;
       
       static ulonglong start_position, stop_position;
       static const longlong stop_position_default= (longlong)(~(my_off_t)0);
      +static ulonglong last_processed_position= 0;
       #define start_position_mot ((my_off_t)start_position)
       #define stop_position_mot  ((my_off_t)stop_position)
       
      @@ -997,6 +998,8 @@ static bool print_row_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev,
         determining how to print.
         @param[in] ev Log_event to process.
         @param[in] pos Offset from beginning of binlog file.
      +  @param[in] ev_end_pos Offset that represents the end of the event in the
      +                        binlog file.
         @param[in] logname Name of input binlog.
       
         @retval ERROR_STOP An error occurred - the program should terminate.
      @@ -1005,7 +1008,7 @@ static bool print_row_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev,
         events to process has been reached and the program should terminate.
       */
       Exit_status process_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev,
      -                          my_off_t pos, const char *logname)
      +                          my_off_t pos, my_off_t ev_end_pos, const char *logname)
       {
         char ll_buff[21];
         Log_event_type ev_type= ev->get_type_code();
      @@ -1461,7 +1464,8 @@ Exit_status process_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev,
         retval= ERROR_STOP;
       end:
         rec_count++;
      -  last_processed_datetime = last_ev_when;
      +  last_processed_datetime= last_ev_when;
      +  last_processed_position= ev_end_pos;
       
         DBUG_PRINT("info", ("end event processing"));
         /*
      @@ -2379,7 +2383,8 @@ static Exit_status handle_event_text_mode(PRINT_EVENT_INFO *print_event_info,
             if (old_off != BIN_LOG_HEADER_SIZE)
               *len= 1;         // fake event, don't increment old_off
           }
      -    Exit_status retval= process_event(print_event_info, ev, old_off, logname);
      +    Exit_status retval= process_event(print_event_info, ev, old_off,
      +                                      old_off + (*len - 1), logname);
           if (retval != OK_CONTINUE)
             DBUG_RETURN(retval);
         }
      @@ -2397,7 +2402,8 @@ static Exit_status handle_event_text_mode(PRINT_EVENT_INFO *print_event_info,
             DBUG_RETURN(ERROR_STOP);
           }
       
      -    retval= process_event(print_event_info, ev, old_off, logname);
      +    retval= process_event(print_event_info, ev, old_off, old_off + (*len - 1),
      +                          logname);
           if (retval != OK_CONTINUE)
           {
             my_close(file,MYF(MY_WME));
      @@ -2796,6 +2802,7 @@ static Exit_status check_header(IO_CACHE* file,
                 */
                 Exit_status retval= process_event(print_event_info,
                                                   new_description_event, tmp_pos,
      +                                            my_b_tell(file),
                                                   logname);
                 if (retval != OK_CONTINUE)
                   return retval;
      @@ -2944,22 +2951,12 @@ static Exit_status dump_local_log_entries(PRINT_EVENT_INFO *print_event_info,
             }
             // else file->error == 0 means EOF, that's OK, we break in this case
       
      -      /*
      -        Emit a warning in the event that we finished processing input
      -        before reaching the boundary indicated by --stop-position.
      -      */
      -      if (((longlong)stop_position != stop_position_default) &&
      -          stop_position > my_b_tell(file))
      -      {
      -          retval = OK_STOP;
      -          warning("Did not reach stop position %llu before "
      -                  "end of input", stop_position);
      -      }
       
             goto end;
           }
      -    if ((retval= process_event(print_event_info, ev, old_off, logname)) !=
      -        OK_CONTINUE)
      +    if ((retval= process_event(print_event_info, ev, old_off, my_b_tell(file),
      +                               logname)) != OK_CONTINUE)
      +
             goto end;
         }
       
      @@ -3135,9 +3132,17 @@ int main(int argc, char** argv)
       
         if (stop_datetime != MY_TIME_T_MAX &&
             stop_datetime > last_processed_datetime)
      -      warning("Did not reach stop datetime '%s' "
      -              "before end of input", stop_datetime_str);
      +    warning("Did not reach stop datetime '%s' before end of input",
      +            stop_datetime_str);
       
      +  /*
      +    Emit a warning in the event that we finished processing input
      +    before reaching the boundary indicated by --stop-position.
      +  */
      +  if (((longlong) stop_position != stop_position_default) &&
      +      stop_position > last_processed_position)
      +    warning("Did not reach stop position %llu before end of input",
      +            stop_position);
         /*
           If enable flashback, need to print the events from the end to the
           beginning
      diff --git a/mysql-test/suite/binlog/r/binlog_mysqlbinlog_warn_stop_position.result b/mysql-test/suite/binlog/r/binlog_mysqlbinlog_warn_stop_position.result
      index 1517fd765ac..40fb791cf02 100644
      --- a/mysql-test/suite/binlog/r/binlog_mysqlbinlog_warn_stop_position.result
      +++ b/mysql-test/suite/binlog/r/binlog_mysqlbinlog_warn_stop_position.result
      @@ -18,6 +18,51 @@ drop table t1;
       # Ensuring file offset of binlog_f2_mid < binlog_f1_end
       #
       #
      +# Test using --read-from-remote-server
      +#
      +connection default;
      +#
      +# --stop-position tests
      +#
      +#  Case 1.a) With one binlog file, a --stop-position before the end of
      +# the file should not result in a warning
      +# MYSQL_BINLOG --read-from-remote-server --stop-position=binlog_f1_pre_rotate binlog_f1_full --result-file=tmp/warn_position_test_file.out 2>&1
      +#
      +#  Case 1.b) With one binlog file, a --stop-position at the exact end of
      +# the file should not result in a warning
      +# MYSQL_BINLOG --read-from-remote-server --stop-position=binlog_f1_end binlog_f1_full --result-file=tmp/warn_position_test_file.out 2>&1
      +#
      +#  Case 1.c) With one binlog file, a --stop-position past the end of the
      +# file should(!) result in a warning
      +# MYSQL_BINLOG --read-from-remote-server --short-form --stop-position=binlog_f1_over_eof binlog_f1_full --result-file=tmp/warn_position_test_file.out 2>&1
      +WARNING: Did not reach stop position <BINLOG_F1_OVER_EOF> before end of input
      +#
      +#  Case 2.a) With two binlog files, a --stop-position targeting b2 which
      +# exists in the size of b1 should:
      +#    1) not provide any warnings
      +#    2) not prevent b2 from outputting its desired events before the
      +#       stop position
      +# MYSQL_BINLOG --read-from-remote-server --stop-position=binlog_f2_mid binlog_f1_full binlog_f2_full --result-file=tmp/warn_position_test_file.out 2>&1
      +include/assert_grep.inc [Ensure all intended GTIDs are present]
      +include/assert_grep.inc [Ensure the next GTID binlogged is _not_ present]
      +#
      +#  Case 2.b) With two binlog files, a --stop-position targeting the end
      +# of binlog 2 should:
      +#    1) not provide any warnings
      +#    2) not prevent b2 from outputting its entire binary log
      +# MYSQL_BINLOG --read-from-remote-server --stop-position=binlog_f2_end binlog_f1_full binlog_f2_full --result-file=tmp/warn_position_test_file.out 2>&1
      +include/assert_grep.inc [Ensure a GTID exists for each transaction]
      +include/assert_grep.inc [Ensure the last GTID binlogged is present]
      +#
      +#  Case 2.c) With two binlog files, a --stop-position targeting beyond
      +# the eof of binlog 2 should:
      +#    1) provide a warning that the stop position was not reached
      +#    2) not prevent b2 from outputting its entire binary log
      +# MYSQL_BINLOG --read-from-remote-server --stop-position=binlog_f2_over_eof binlog_f1_full binlog_f2_full --result-file=tmp/warn_position_test_file.out 2>&1
      +WARNING: Did not reach stop position <BINLOG_F2_OVER_EOF> before end of input
      +include/assert_grep.inc [Ensure a GTID exists for each transaction]
      +#
      +#
       # Test using local binlog files
       #
       connection default;
      diff --git a/mysql-test/suite/binlog/t/binlog_mysqlbinlog_warn_stop_position.inc b/mysql-test/suite/binlog/t/binlog_mysqlbinlog_warn_stop_position.inc
      index 7ecf5c1098a..878de8ef2b5 100644
      --- a/mysql-test/suite/binlog/t/binlog_mysqlbinlog_warn_stop_position.inc
      +++ b/mysql-test/suite/binlog/t/binlog_mysqlbinlog_warn_stop_position.inc
      @@ -50,7 +50,7 @@ if ($use_local_files)
       --echo #
       --echo #  Case 1.c) With one binlog file, a --stop-position past the end of the
       --echo # file should(!) result in a warning
      ---let $binlog_f1_over_eof= `SELECT $binlog_f1_end + 10`
      +--let $binlog_f1_over_eof= `SELECT $binlog_f1_end + 1`
       --echo # MYSQL_BINLOG $PARAM_READ_FROM_REMOTE_OUT --short-form --stop-position=binlog_f1_over_eof binlog_f1_full --result-file=$binlog_out_relpath 2>&1
       --replace_result $binlog_f1_over_eof <BINLOG_F1_OVER_EOF>
       --exec $MYSQL_BINLOG $PARAM_READ_FROM_REMOTE --short-form --stop-position=$binlog_f1_over_eof $binlog_f1_full --result-file=$binlog_out 2>&1
      @@ -102,7 +102,7 @@ if ($use_local_files)
       --echo # the eof of binlog 2 should:
       --echo #    1) provide a warning that the stop position was not reached
       --echo #    2) not prevent b2 from outputting its entire binary log
      ---let $binlog_f2_over_eof= `SELECT $binlog_f2_end + 10`
      +--let $binlog_f2_over_eof= `SELECT $binlog_f2_end + 1`
       --echo # MYSQL_BINLOG $PARAM_READ_FROM_REMOTE_OUT --stop-position=binlog_f2_over_eof binlog_f1_full binlog_f2_full --result-file=$binlog_out_relpath 2>&1
       --replace_result $binlog_f2_over_eof <BINLOG_F2_OVER_EOF>
       --exec $MYSQL_BINLOG $PARAM_READ_FROM_REMOTE --stop-position=$binlog_f2_over_eof $binlog_f1_full $binlog_f2_full --result-file=$binlog_out 2>&1
      diff --git a/mysql-test/suite/binlog/t/binlog_mysqlbinlog_warn_stop_position.test b/mysql-test/suite/binlog/t/binlog_mysqlbinlog_warn_stop_position.test
      index c1028a23404..34ec9e67579 100644
      --- a/mysql-test/suite/binlog/t/binlog_mysqlbinlog_warn_stop_position.test
      +++ b/mysql-test/suite/binlog/t/binlog_mysqlbinlog_warn_stop_position.test
      @@ -64,13 +64,12 @@ if ($binlog_f2_mid > $binlog_f1_end)
         --die Mid point chosen to end in binlog 2 does not exist in earlier binlog
       }
       
      -#--echo #
      -#--echo #
      -#--echo # Test using --read-from-remote-server
      -#--echo #
      -#--let $use_local_files= 0
      -#--emit warning is not supported by --read-from-remote-server now
      -#--source binlog_mysqlbinlog_warn_stop_position.inc
      +--echo #
      +--echo #
      +--echo # Test using --read-from-remote-server
      +--echo #
      +--let $use_local_files= 0
      +--source binlog_mysqlbinlog_warn_stop_position.inc
       
       --echo #
       --echo #
      

      Attachments

        Issue Links

          Activity

            People

              bnestere Brandon Nesterenko
              bnestere Brandon Nesterenko
              Votes:
              0 Vote for this issue
              Watchers:
              2 Start watching this issue

              Dates

                Created:
                Updated:

                Git Integration

                  Error rendering 'com.xiplink.jira.git.jira_git_plugin:git-issue-webpanel'. Please contact your Jira administrators.