From 2da2b6e23ffa0a953f609012c87ba33ccece11b3 Mon Sep 17 00:00:00 2001 From: Zsolt Parragi Date: Wed, 8 May 2019 13:40:38 -0700 Subject: [PATCH] FB8-67: Publish git hash in db log and via global status (#991) (#991) Summary: Jira ticket: https://jira.percona.com/browse/FB8-67 Reference commit: https://github.com/facebook/mysql-5.6/commit/e4f22e0 Reference commit: https://github.com/facebook/mysql-5.6/commit/f00855b Reference commit: https://github.com/facebook/mysql-5.6/commit/92bec02 Reference commit: https://github.com/facebook/mysql-5.6/commit/408ea32ed2 Rocksdb hash and date is set to "none" until rocksdb is added. Pull Request resolved: https://github.com/facebook/mysql-5.6/pull/991 Reviewed By: lloyd Differential Revision: D14603932 Pulled By: lth --- cmake/githash.pl | 214 ++++++++++++++++++++++++++++ share/messages_to_clients.txt | 254 ++++++++++++++++++++++++++++++++++ sql/CMakeLists.txt | 33 +++++ sql/mysql_githash.h.in | 31 +++++ sql/mysqld.cc | 19 +++ 5 files changed, 551 insertions(+) create mode 100755 cmake/githash.pl create mode 100644 sql/mysql_githash.h.in diff --git a/cmake/githash.pl b/cmake/githash.pl new file mode 100755 index 000000000000..6ac3379ca297 --- /dev/null +++ b/cmake/githash.pl @@ -0,0 +1,214 @@ +#!/usr/bin/perl -w + +# +# Copyright (c) 2016 Facebook, Inc. +# All rights reserved +# + +# +# Git hash/date header file generator +# + +use strict; + +use Cwd; +use File::Copy qw(copy); +use Getopt::Long qw(GetOptions); +use Pod::Usage qw(pod2usage); + +sub check_all_or_none { + my $bitmask_a = 0; + $bitmask_a = $bitmask_a | 1 if shift ne ""; + $bitmask_a = $bitmask_a | 2 if shift ne ""; + + my $bitmask_b = 0; + $bitmask_b = $bitmask_b | 1 if shift ne ""; + $bitmask_b = $bitmask_b | 2 if shift ne ""; + + pod2usage("You must specify both or none of the mysql git hash and date") + unless ($bitmask_a == 0 || $bitmask_a == 3); + + pod2usage("You must specify both or none of the rocksdb git hash and date") + unless ($bitmask_b == 0 || $bitmask_b == 3); +} + +# Function to retrieve the git hash and date from a repository +sub git_hash_and_date { + my $subdir = shift; + my $orgdir = Cwd::getcwd; + + # Switch directories to the specified one + chdir($subdir) or die "Can't change directory to '$subdir'"; + + # Get the hash and date from the most recent revision in the repository + my $git_cmd = "git log -1 --format=\"%H;%cI\""; + open (my $log, "$git_cmd |") or die "Can't run $git_cmd"; + + my $githash = ""; + my $gitdate = ""; + + # Loop through all the lines - we should only have one line + while (<$log>) { + # Search for a line that has a hash, a semicolon and the date + if (/^([0-9a-f]{7,40});(.*)\n/) { + die "Unexpected multiple matching log lines" unless !length($githash); + $githash = $1; + $gitdate = $2; + } + } + + # Make sure we got something + die "No matching log lines" unless length($githash); + + # Close the input and switch back to the original subdirectory + close($log); + chdir($orgdir); + + # Return the found hash and date + return ($githash, $gitdate); +} + +# main function +sub main { + my $root = ""; + my $infile = ""; + my $mysql_git_hash = ""; + my $mysql_git_date = ""; + my $rocksdb_git_hash = ""; + my $rocksdb_git_date = ""; + my $help = 0; + + # Get the parameters + GetOptions( + 'help|?' => \$help, + 'git_root=s' => \$root, + 'file=s' => \$infile, + 'mysql_githash=s' => \$mysql_git_hash, + 'mysql_gitdate=s' => \$mysql_git_date, + 'rocksdb_githash=s' => \$rocksdb_git_hash, + 'rocksdb_gitdate=s' => \$rocksdb_git_date + ) or pod2usage(); + + # Validate the parameters + pod2usage(-verbose => 1) if $help; + pod2usage("missing required parameter --git_root") if $root eq ""; + pos2usage("missing required parameter --file") if $infile eq ""; + check_all_or_none($mysql_git_hash, $mysql_git_date, + $rocksdb_git_hash, $rocksdb_git_date); + + my $rocksdb = "$root/rocksdb"; + my $outfile = "$infile.tmp"; + + if ($mysql_git_hash eq "") { + # retrieve the git hash and date for the main repository + ($mysql_git_hash, $mysql_git_date) = git_hash_and_date $root; + } + if ($rocksdb_git_hash eq "") { + if (-d $rocksdb) { + # retrieve the git hash and date for the rocksdb submodule + ($rocksdb_git_hash, $rocksdb_git_date) = git_hash_and_date $rocksdb; + } else { + $rocksdb_git_hash="none"; + $rocksdb_git_date="none"; + } + } + + # Open the user's file for reading and a temporary file for writing + open(my $in, "<", $infile) or die "Could not open '$infile'"; + open(my $out, ">", $outfile) or die "Could not create '$outfile'"; + + # For each line, see if we can replace anything + while (<$in>) { + s/\@MYSQL_GIT_HASH\@/$mysql_git_hash/g; + s/\@MYSQL_GIT_DATE\@/$mysql_git_date/g; + s/\@ROCKSDB_GIT_HASH\@/$rocksdb_git_hash/g; + s/\@ROCKSDB_GIT_DATE\@/$rocksdb_git_date/g; + print $out $_; + } + + # Close both files + close $in; + close $out; + + # Copy the temporary file to the original and then delete it + copy $outfile, $infile or + die "Unable to copy temp file ($outfile) on top of original ($infile)"; + unlink $outfile; +} + +main(@ARGV); + +=head1 NAME + +githash.pl - Generate the mysql_githash.h file for MySQL + +=head1 SYNOPSIS + +githash.pl [options] + + Options: + --help Brief help message + --git_root= Path to the root of the MySQL git repository + --file= File to update with the git hashes and dates + --mysql_githash= Optional git hash to use for MySQL + --mysql_gitdate= Optional git date to use for MySQL + --rocksdb_githash= Optional git hash to use for RocksDB + --rocksdb_gitdate= Optional git date to use for RocksDB + +=head1 OPTIONS + +=over 4 + +=item B<--help> + +Print a brief help message list all the options and exits + +=item B<--git_root> + +Supplies the path to the root git repository. If the optional hashes and +dates are not supplied this script will call 'git log' in this location to get +the hashes and dates. + +=item B<--file> + +Specifies the file to update. This file will be scanned for the following +markers, which will be replaced by the corresponding hash or date: + + @MYSQL_GIT_HASH@ + @MYSQL_GIT_DATE@ + @ROCKSDB_GIT_HASH@ + @ROCKSDB_GIT_DATE@ + +=item B<--mysql_githash> + +Optional git hash for the MySQL repository. If this is not specified the +script will use "git log" to query the repository. If one of the hashes or +dates is supplied, all of them must be supplied. + +=item B<--mysql_gitdate> + +Optional git date for the MySQL repository. If this is not specified the +script will use "git log" to query the repository. If one of the hashes or +dates is supplied, all of them must be supplied. + +=item B<--rocksdb_githash> + +Optional git hash for the RocksDB repository. If this is not specified the +script will use "git log" to query the repository. If one of the hashes or +dates is supplied, all of them must be supplied. + +=item B<--rocksdb_gitdate> + +Optional git date for the RocksDB repository. If this is not specified the +script will use "git log" to query the repository. If one of the hashes or +dates is supplied, all of them must be supplied. + +=back + +=head1 DESCRIPTION + +B will take the git hashes and dates (either supplied via +parameters or retrieved from 'git log') and update the specified file with the +values. + +=cut diff --git a/share/messages_to_clients.txt b/share/messages_to_clients.txt index e010b378799c..c756d1b8ded4 100644 --- a/share/messages_to_clients.txt +++ b/share/messages_to_clients.txt @@ -9992,3 +9992,257 @@ reserved-error-section 5000 5999 # same directory as this file for more # information. ################################################################################ + + +# +# FB MySQL error codes begin from 50000 +# + +start-error-number 50000 + +ER_PLACEHOLDER_50000 + eng "Placeholder" + +ER_PLACEHOLDER_50001 + eng "Placeholder" + +ER_PLACEHOLDER_50002 + eng "Placeholder" + +ER_PLACEHOLDER_50003 + eng "Placeholder" + +ER_PLACEHOLDER_50004 + eng "Placeholder" + +ER_PLACEHOLDER_50005 + eng "Placeholder" + +ER_PLACEHOLDER_50006 + eng "Placeholder" + +ER_PLACEHOLDER_50007 + eng "Placeholder" + +ER_PLACEHOLDER_50008 + eng "Placeholder" + +ER_PLACEHOLDER_50009 + eng "Placeholder" + +ER_PLACEHOLDER_50010 + eng "Placeholder" + +ER_PLACEHOLDER_50011 + eng "Placeholder" + +ER_PLACEHOLDER_50012 + eng "Placeholder" + +ER_PLACEHOLDER_50013 + eng "Placeholder" + +ER_PLACEHOLDER_50014 + eng "Placeholder" + +ER_PLACEHOLDER_50015 + eng "Placeholder" + +ER_PLACEHOLDER_50016 + eng "Placeholder" + +ER_PLACEHOLDER_50017 + eng "Placeholder" + +ER_PLACEHOLDER_50018 + eng "Placeholder" + +ER_PLACEHOLDER_50019 + eng "Placeholder" + +ER_PLACEHOLDER_50020 + eng "Placeholder" + +ER_PLACEHOLDER_50021 + eng "Placeholder" + +ER_PLACEHOLDER_50022 + eng "Placeholder" + +ER_GIT_HASH + eng "%s git hash: %s - %s" + +ER_PLACEHOLDER_50024 + eng "Placeholder" + +ER_PLACEHOLDER_50025 + eng "Placeholder" + +ER_PLACEHOLDER_50026 + eng "Placeholder" + +ER_PLACEHOLDER_50027 + eng "Placeholder" + +ER_PLACEHOLDER_50028 + eng "Placeholder" + +ER_PLACEHOLDER_50029 + eng "Placeholder" + +ER_PLACEHOLDER_50030 + eng "Placeholder" + +ER_PLACEHOLDER_50031 + eng "Placeholder" + +ER_PLACEHOLDER_50032 + eng "Placeholder" + +ER_PLACEHOLDER_50033 + eng "Placeholder" + +ER_PLACEHOLDER_50034 + eng "Placeholder" + +ER_PLACEHOLDER_50035 + eng "Placeholder" + +ER_PLACEHOLDER_50036 + eng "Placeholder" + +ER_PLACEHOLDER_50037 + eng "Placeholder" + +ER_PLACEHOLDER_50038 + eng "Placeholder" + +ER_PLACEHOLDER_50039 + eng "Placeholder" + +ER_KEYS_OUT_OF_ORDER + eng "Keys are out order during bulk load" + +ER_OVERLAPPING_KEYS + eng "Bulk load rows overlap existing rows" + +ER_REQUIRE_ROW_BINLOG_FORMAT + eng "Can't execute updates on master with binlog_format != ROW." + +ER_ISOLATION_MODE_NOT_SUPPORTED + eng "MyRocks supports only READ COMMITTED and REPEATABLE READ isolation levels. Please change from current isolation level %s" + +ER_ON_DUPLICATE_DISABLED + eng "When unique checking is disabled in MyRocks, INSERT,UPDATE,LOAD statements with clauses that update or replace the key (i.e. INSERT ON DUPLICATE KEY UPDATE, REPLACE) are not allowed. Query: %s" + +ER_UPDATES_WITH_CONSISTENT_SNAPSHOT + eng "Can't execute updates when you started a transaction with START TRANSACTION WITH CONSISTENT|SHARED|EXISTING [ROCKSDB] SNAPSHOT." + +ER_ROLLBACK_ONLY + eng "This transaction was rolled back and cannot be committed. Only supported operation is to roll it back, so all pending changes will be discarded. Please restart another transaction." + +ER_ROLLBACK_TO_SAVEPOINT + eng "MyRocks currently does not support ROLLBACK TO SAVEPOINT if modifying rows." + +ER_ISOLATION_LEVEL_WITH_CONSISTENT_SNAPSHOT + eng "Only REPEATABLE READ isolation level is supported for START TRANSACTION WITH CONSISTENT|SHARED|EXISTING SNAPSHOT in RocksDB Storage Engine." + +ER_UNSUPPORTED_COLLATION + eng "Unsupported collation on string indexed column %s.%s Use binary collation (%s)." + +ER_METADATA_INCONSISTENCY + eng "Table '%s' does not exist, but metadata information exists inside MyRocks. This is a sign of data inconsistency." + +ER_KEY_CREATE_DURING_ALTER + eng "MyRocks failed creating new key definitions during alter." + +ER_SK_POPULATE_DURING_ALTER + eng "MyRocks failed populating secondary key during alter." + +ER_CF_DIFFERENT + eng "Column family ('%s') flag (%d) is different from an existing flag (%d). Assign a new CF flag, or do not change existing CF flag." + +ER_RDB_TTL_UNSUPPORTED + eng "TTL support is currently disabled when table has a hidden PK." + +ER_RDB_TTL_COL_FORMAT + eng "TTL column (%s) in MyRocks must be an unsigned non-null 64-bit integer, exist inside the table, and have an accompanying ttl duration." + +ER_RDB_TTL_DURATION_FORMAT + eng "TTL duration (%s) in MyRocks must be an unsigned non-null 64-bit integer." + +ER_PER_INDEX_CF_DEPRECATED + eng "The per-index column family option has been deprecated" + +ER_NP_PROCEDURE_EXISTS + eng "Native procedure '%-.192s' exists." + +ER_NP_UNKNOWN_PROCEDURE + eng "Unknown native procedure. '%-.192s'" + +ER_NP_WRONG_ARGUMENTS + eng "Incorrect arguments to native procedure. (query '%-.192s')" + +ER_NP_FAILED + eng "Native procedure failed. (code: %d, msg: '%-.192s', query '%-.192s')" + +ER_CANT_SET_DEPENDENCY_REPLICATION_WITHOUT_IDEMPOTENT_RECOVERY + eng "@@GLOBAL.MTS_DEPENDENCY_REPLICATION cannot be set when @@GLOBAL.SLAVE_USE_IDEMPOTENT_FOR_RECOVERY = NO" + +ER_PLACEHOLDER_50063 + eng "Placeholder" + +ER_PLACEHOLDER_50064 + eng "Placeholder" + +ER_PLACEHOLDER_50065 + eng "Placeholder" + +ER_UPDATES_WITH_EXPLICIT_SNAPSHOT + eng "Can't execute updates when an explicit snapshot is associated with the connection using CREATE|ATTACH EXPLICIT [ENGINE] SNAPSHOT" + +# RPC detached session errors start here +ER_PLACEHOLDER_50067 + eng "Placeholder" + +ER_PLACEHOLDER_50068 + eng "Placeholder" + +ER_PLACEHOLDER_50069 + eng "Placeholder" + +ER_PLACEHOLDER_50070 + eng "Placeholder" + +ER_PLACEHOLDER_50071 + eng "Placeholder" + +ER_PLACEHOLDER_50072 + eng "Placeholder" + +ER_PLACEHOLDER_50073 + eng "Placeholder" + +ER_PLACEHOLDER_50074 + eng "Placeholder" + +ER_PLACEHOLDER_50075 + eng "Placeholder" + +ER_PLACEHOLDER_50076 + eng "Placeholder" + +ER_PLACEHOLDER_50077 + eng "Placeholder" + +ER_CF_DROPPED + eng "Column family ('%s') is being dropped." + +ER_CANT_DROP_CF + eng "Cannot drop Column family ('%s') because it is in use or does not exist." + +# +# End of 8.0 FB MySQL error messages. +# (Please read comments from the header of this section before adding error +# message here) +# diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index c64c9b309d3a..caf86671f3bb 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -47,6 +47,7 @@ SET(GEN_SOURCES ${CMAKE_CURRENT_BINARY_DIR}/sql_hints.yy.h ${CMAKE_CURRENT_BINARY_DIR}/sql_hints.yy.cc ${CMAKE_CURRENT_BINARY_DIR}/lex_hash.h + ${CMAKE_CURRENT_BINARY_DIR}/mysql_githash.h ) SET(GEN_DIGEST_SOURCES ${CMAKE_CURRENT_BINARY_DIR}/lex_token.h @@ -1468,3 +1469,35 @@ ADD_CUSTOM_TARGET(distclean ADD_CUSTOM_TARGET(show-dist-name COMMAND ${CMAKE_COMMAND} -E echo "${CPACK_PACKAGE_FILE_NAME}" ) + +# If the repository was modified (via push or pull) or if the +# mysql_githash.h.in changed rebuild the mysql_githash.h file +SET(GITHASH_HEADER_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/mysql_githash.h.in) +SET(GITHASH_HEADER ${CMAKE_CURRENT_BINARY_DIR}/mysql_githash.h) + + +SET(GITHASH_DEPENDS ${GITHASH_HEADER_SOURCE}) +SET(GITHASH_PERL_OPTS --git_root=${CMAKE_SOURCE_DIR} --file=${GITHASH_HEADER}) + +IF(DEFINED MYSQL_GITHASH) + # If the hashes and dates are passed in add them to the parameters for + # the perl script + SET(GITHASH_PERL_OPTS ${GITHASH_PERL_OPTS} + --mysql_githash=${MYSQL_GITHASH} + --mysql_gitdate=${MYSQL_GITDATE} + --rocksdb_githash=${ROCKSDB_GITHASH} + --rocksdb_gitdate=${ROCKSDB_GITDATE}) +ELSE() + # Otherwise we want to add the .git/logs/HEAD file to the list of + # dependencies + SET(GITHASH_DEPENDS ${GITHASH_DEPENDS} ${CMAKE_SOURCE_DIR}/.git/logs/HEAD) +ENDIF() + +SET(GITHASH_PERL ${CMAKE_SOURCE_DIR}/cmake/githash.pl) +ADD_CUSTOM_COMMAND(OUTPUT ${GITHASH_HEADER} + COMMAND ${CMAKE_COMMAND} -E copy ${GITHASH_HEADER_SOURCE} ${GITHASH_HEADER} + COMMAND ${CMAKE_COMMAND} -E echo ${GITHASH_PERL} ${GITHASH_PERL_OPTS} + COMMAND perl ${GITHASH_PERL} ${GITHASH_PERL_OPTS} + DEPENDS ${GITHASH_DEPENDS} + VERBATIM +) diff --git a/sql/mysql_githash.h.in b/sql/mysql_githash.h.in new file mode 100644 index 000000000000..c202f66312a3 --- /dev/null +++ b/sql/mysql_githash.h.in @@ -0,0 +1,31 @@ +/* + Copyright (c) 2016, Facebook, Inc. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +/* + Do not hand edit this file. It is generated as part of the build + process and contains git hashes and dates for MySQL and RocksDB +*/ +#ifndef MYSQL_GITHASH_H +#define MYSQL_GITHASH_H + +#define MYSQL_GIT_HASH "@MYSQL_GIT_HASH@" +#define MYSQL_GIT_DATE "@MYSQL_GIT_DATE@" + +#define ROCKSDB_GIT_HASH "@ROCKSDB_GIT_HASH@" +#define ROCKSDB_GIT_DATE "@ROCKSDB_GIT_DATE@" + +#endif diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 7ac7e6c99a5a..b85f554f9cb2 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -820,6 +820,7 @@ MySQL clients support the protocol: #endif #include "my_openssl_fips.h" // OPENSSL_ERROR_LENGTH, set_fips_mode #include "sql/rpl_async_conn_failover_configuration_propagation.h" +#include "sql/mysql_githash.h" #include "sql/rpl_filter.h" #include "sql/rpl_gtid.h" #include "sql/rpl_gtid_persist.h" // Gtid_table_persistor @@ -1168,6 +1169,12 @@ bool temptable_use_mmap; static char compiled_default_collation_name[] = MYSQL_DEFAULT_COLLATION_NAME; static bool binlog_format_used = false; +/* MySQL git hashes and dates */ +static char git_hash[] = MYSQL_GIT_HASH; +static char git_date[] = MYSQL_GIT_DATE; +static char rocksdb_git_hash[] = ROCKSDB_GIT_HASH; +static char rocksdb_git_date[] = ROCKSDB_GIT_DATE; + LEX_STRING opt_init_connect, opt_init_replica; /* Global variables */ @@ -8214,6 +8221,12 @@ int mysqld_main(int argc, char **argv) create_compress_gtid_table_thread(); + // NO_LINT_DEBUG + sql_print_information(ER_DEFAULT(ER_GIT_HASH), "MySQL", git_hash, git_date); + // NO_LINT_DEBUG + sql_print_information(ER_DEFAULT(ER_GIT_HASH), "RocksDB", rocksdb_git_hash, + rocksdb_git_date); + LogEvent() .type(LOG_TYPE_ERROR) .subsys(LOG_SUBSYSTEM_TAG) @@ -9737,6 +9750,12 @@ SHOW_VAR status_vars[] = { SHOW_LONGLONG, SHOW_SCOPE_GLOBAL}, {"Flush_commands", (char *)&refresh_version, SHOW_LONG_NOFLUSH, SHOW_SCOPE_GLOBAL}, + {"Git_hash", (char *)git_hash, SHOW_CHAR, SHOW_SCOPE_GLOBAL}, + {"Git_date", (char *)git_date, SHOW_CHAR, SHOW_SCOPE_GLOBAL}, + {"Rocksdb_git_hash", (char *)rocksdb_git_hash, SHOW_CHAR, + SHOW_SCOPE_GLOBAL}, + {"Rocksdb_git_date", (char *)rocksdb_git_date, SHOW_CHAR, + SHOW_SCOPE_GLOBAL}, {"Global_connection_memory", (char *)&show_global_mem_counter, SHOW_FUNC, SHOW_SCOPE_GLOBAL}, {"Handler_commit", (char *)offsetof(System_status_var, ha_commit_count),