#!/bin/bash

# Copyright (c) 2012,2013 Oracle and/or its affiliates. All rights reserved.

# Use is subject to license terms.

#

# This program is free software; you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation; version 2 of the License.

#

# This program is distributed in the hope that it will be useful, but

# WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU

# General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program; if not, write to the Free Software

# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301

# USA

# In active development: 2012-2018

# This program has been used to reduce thousands of SQL based testcases from tens of thousands of lines to < 10 lines. Learn more at;

# https://www.percona.com/blog/2014/09/03/reducer-sh-a-powerful-mysql-test-case-simplificationreducer-tool/

# https://www.percona.com/blog/2015/07/21/mysql-qa-episode-7-single-threaded-reducer-sh-reducing-testcases-for-beginners

# https://www.percona.com/blog/2015/07/23/mysql-qa-episode-8-reducing-testcases-engineers-tuning-reducer-sh/

# https://www.percona.com/blog/2015/07/28/mysql-qa-episode-9-reducing-testcases-experts-multi-threaded-reducer-sh/

# https://www.percona.com/blog/2015/07/31/mysql-qa-episode-10-reproducing-simplifying-get-right/

# https://www.percona.com/blog/2015/03/17/free-mysql-qa-and-bash-linux-training-series/

# ======== Dev Contacts

# Main developer: Roel Van de Paar <roel A.T vandepaar D.O.T com>

# With contributions from & thanks to: Andrew Dalgleish, Ramesh Sivaraman, Tomislav Plavcic

# With thanks to the team at Oracle for open sourcing the original internal version

# ======== User configurable variables section (see 'User configurable variable reference' below for more detail)

# === Basic options

INPUTFILE= # The SQL file to be reduced. This can also be given as the first option to reducer.sh. Do not use double quotes

MODE=4 # Required. Most often used modes: 4=Any crash (TEXT not required), 3=Search for a specific TEXT in mysqld error log, 2=Idem, but in client log

TEXT="somebug" # The text string you want reducer to search for, in specific locations depending on the MODE selected. Regex capable. Use with MODEs=1,2,3,5,6,7,8

WORKDIR_LOCATION=1 # 0: use /tmp (disk bound) | 1: use tmpfs (default) | 2: use ramfs (needs setup) | 3: use storage at WORKDIR_M3_DIRECTORY

WORKDIR_M3_DIRECTORY="/ssd" # Only relevant if WORKDIR_LOCATION is set to 3, use a specific directory/mount point

MYEXTRA="--no-defaults --log-output=none --sql_mode=ONLY_FULL_GROUP_BY" # mysqld options to be used (and reduced). Note: TokuDB plugin loading is checked/done automatically

MYINIT="" # Extra options to pass to mysqld AND at data dir init time. See pquery-run-*.conf for more info

BASEDIR="/sda/PS051018-percona-server-8.0.12-1-linux-x86_64-debug" # Path to the MySQL BASE directory to be used

DISABLE_TOKUDB_AUTOLOAD=0 # On/Off (1/0) Prevents mysqld startup issues when using standard MySQL server (i.e. no TokuDB available) with a testcase containing TokuDB SQL

SCRIPT_PWD=$(cd `dirname $0` && pwd) # script location to access storage engine plugin sql file.

# === Sporadic testcases # Used when testcases prove to be sporadic *and* fail to reduce using basic methods

FORCE_SKIPV=0 # On/Off (1/0) Forces verify stage to be skipped (auto-enables FORCE_SPORADIC)

FORCE_SPORADIC=0 # On/Off (1/0) Forces issue to be treated as sporadic

# === True Multi-Threaded # True multi-threaded testcase reduction (only program in the world that does this) based on random replay (auto-covers sporadic testcases)

PQUERY_MULTI=0 # On/off (1/0) Enables true multi-threaded testcase reduction based on random replay (auto-enables PQUERY_MOD)

# === Reduce startup issues # Reduces startup issues. This will only work if a clean start (mysqld --no-defaults) works correctly; otherwise template creation will fail also

REDUCE_STARTUP_ISSUES=0 # Default/normal use: 0. Set to 1 to reduce mysqld startup (ie. failing mysqld --option etc.) issues (with SQL replay but without SQL simplication)

# === Reduce GLIBC/SS crashes # Remember that if you use REDUCE_GLIBC_OR_SS_CRASHES=1 with MODE=3, then the console/typescript log is searched for TEXT, not the mysqld error log

REDUCE_GLIBC_OR_SS_CRASHES=0 # Default/normal use: 0. Set to 1 to reduce testcase based on a GLIBC crash or stack smash being detected. MODE=3 (TEXT) and MODE=4 (all) supported

SCRIPT_LOC=/usr/bin/script # The script binary (sudo yum install util-linux) is required for reducing GLIBC crashes

# === Hang issues # For catching hang issues (both in normal runtime as well as during shutdown). Must set MODE=0 for this option to become active

TIMEOUT_CHECK=600 # When MODE=0 is used, this specifies the nr of seconds to be used as a timeout. Do not set too small (eg. >600 sec). See examples in help below.

# === Timeout mysqld # Uncommonly used option. Used to terminate (timeout) mysqld after x seconds, while still checking for MODE=2/3 TEXT. See examples in help below.

TIMEOUT_COMMAND="" # A specific command, executed as a prefix to mysqld. For example, TIMEOUT_COMMAND="timeout --signal=SIGKILL 10m"

# === Advanced options # Note: SLOW_DOWN_CHUNK_SCALING is of beta quality. It works, but it may affect chunk scaling somewhat negatively in some cases

SLOW_DOWN_CHUNK_SCALING=0 # On/off (1/0) If enabled, reducer will slow down it's internal chunk size scaling (also see SLOW_DOWN_CHUNK_SCALING_NR)

SLOW_DOWN_CHUNK_SCALING_NR=3 # Slow down chunk size scaling (both for chunk reductions and increases) by not modifying the chunk for this number of trials. Default=3

USE_TEXT_STRING=0 # On/off (1/0) If enabled, when using MODE=3, this uses text_string.sh (percona-qa) instead of searching the entire error log. No effect otherwise

TEXT_STRING_LOC=~/percona-qa/text_string.sh # text_string.sh binary in percona-qa. To get this script use: cd ~; git clone https://github.com/Percona-QA/percona-qa.git

# === Expert options

MULTI_THREADS=10 # Do not change (default=10), unless you fully understand the change (x mysqld servers + 1 mysql or pquery client each)

MULTI_THREADS_INCREASE=5 # Do not change (default=5), unless you fully understand the change (increase of above, both for std and PQUERY_MULTI)

MULTI_THREADS_MAX=50 # Do not change (default=50), unless you fully understand the change (maximum number of MULTI threads, both for std and PQUERY_MULTI)

PQUERY_EXTRA_OPTIONS="" # Do not change (default=""), unless you fully understand the change (adds extra options to pquery replay, used for QC trials)

PQUERY_MULTI_THREADS=3 # Do not change (default=3), unless you fully understand the change (x mysqld servers + 1 pquery client with x threads)

PQUERY_MULTI_CLIENT_THREADS=30 # Do not change (default=30), unless you fully understand the change (x [client] threads mentioned above)

PQUERY_MULTI_QUERIES=400000 # Do not change (default=400000), unless you fully understand the change (queries to be executed per client per trial)

PQUERY_REVERSE_NOSHUFFLE_OPT=0 # Do not change (default=0), unless you fully understand the change (reverses --no-shuffle into shuffle and vice versa)

# On/Off (1/0) (Default=0: --no-shuffle is used for standard pquery replay, shuffle is used for PQUERY_MULTI. =1 reverses this)

# === pquery options # Note: only relevant if pquery is used for testcase replay, ref PQUERY_MOD and PQUERY_MULTI

PQUERY_MOD=0 # On/Off (1/0) Enable to use pquery instead of the mysql CLI. pquery binary (as set in PQUERY_LOC) must be available

PQUERY_LOC=~/percona-qa/pquery/pquery # The pquery binary in percona-qa. To get this binary use: cd ~; git clone https://github.com/Percona-QA/percona-qa.git

# === Other options # The options are not often changed

CLI_MODE=0 # When using the CLI; 0: sent SQL using a pipe, 1: sent SQL using --execute="SOURCE ..." command, 2: sent SQL using redirection (mysql < input.sql)

ENABLE_QUERYTIMEOUT=0 # On/Off (1/0) Enable the Query Timeout function (which also enables and uses the MySQL event scheduler)

QUERYTIMEOUT=90 # Query timeout in sec. Note: queries terminated by the query timeout did not fully replay, and thus overall issue reproducibility may be affected

LOAD_TIMEZONE_DATA=0 # On/Off (1/0) Enable loading Timezone data into the database (mainly applicable for RQG runs) (turned off by default=0 since 26.05.2016)

STAGE1_LINES=90 # Proceed to stage 2 when the testcase is less then x lines (auto-reduced when FORCE_SPORADIC or FORCE_SKIPV are active)

SKIPSTAGEBELOW=0 # Usually not changed (default=0), skips stages below and including this stage

SKIPSTAGEABOVE=99 # Usually not changed (default=99), skips stages above and including this stage

FORCE_KILL=0 # On/Off (1/0) Enable to forcefully kill mysqld instead of using mysqladmin shutdown etc. Auto-disabled for MODE=0.

# === Percona XtraDB Cluster

PXC_MOD=0 # On/Off (1/0) Enable to reduce testcases using a Percona XtraDB Cluster. Auto-enables PQUERY_MODE=1

PXC_ISSUE_NODE=0 # The node on which the issue would/should show (0,1,2 or 3) (default=0 = check all nodes to see if issue occured)

ENCRYPTION_RUN=0 # On/Off (1/0) Enable pxc encryption options

WSREP_PROVIDER_OPTIONS="" # wsrep_provider_options to be used (and reduced).

# === MySQL Group Replication

GRP_RPL_MOD=0 # On/Off (1/0) Enable to reduce testcases using MySQL Group Replication. Auto-enables PQUERY_MODE=1

GRP_RPL_ISSUE_NODE=0 # The node on which the issue would/should show (0,1,2 or 3) (default=0 = check all nodes to see if issue occured)

# === MODE=5 Settings # Only applicable when MODE5 is used

MODE5_COUNTTEXT=1 # Number of times the text should appear (default=minimum=1). Currently only used for MODE=5

MODE5_ADDITIONAL_TEXT="" # An additional string to look for in the CLI output when using MODE 5. When not using this set to "" (=default)

MODE5_ADDITIONAL_COUNTTEXT=1 # Number of times the additional text should appear (default=minimum=1). Only used for MODE=5 and where MODE5_ADDITIONAL_TEXT is not ""

# === Old ThreadSync options # No longer commonly used

TS_TRXS_SETS=0

TS_DBG_CLI_OUTPUT=0

TS_DS_TIMEOUT=10

TS_VARIABILITY_SLEEP=1

# ======== Machine configurable variables section: DO NOT REMOVE THIS

#VARMOD# < please do not remove this, it is here as a marker for other scripts (including reducer itself) to auto-insert settings

# ==== MySQL command line (CLI) output TEXT search examples

#TEXT= "\| 0 \| 7 \|" # Example of how to set TEXT for MySQL CLI output (for MODE=2 or 5)

#TEXT= "\| i \|" # Idem, text instead of number (text is left-aligned, numbers are right-aligned in the MySQL CLI output)

# ======== User configurable variable reference

# - INPUTFILE: the SQL trace to be reduced by reducer.sh. This can also be given as the fisrt option to reducer.sh (i.e. $ ./reducer.sh {inputfile.sql})

# - MODE:

# - MODE=0: Timeout testing (server hangs, shutdown issues, excessive command duration etc.) (set TIMEOUT_CHECK)

# - MODE=1: Valgrind output testing (set TEXT)

# - MODE=2: mysql CLI (Command Line Interface, i.e. the mysql client)/pquery client output testing (set TEXT)

# - MODE=3: mysqld error output log or console/typescript log (when REDUCE_GLIBC_OR_SS_CRASHES=1) testing (set TEXT)

# - MODE=4: Crash or GLIBC crash (when REDUCE_GLIBC_OR_SS_CRASHES=1) testing

# - MODE=5 [BETA]: MTR testcase reduction (set TEXT) (Can also be used for multi-occurence CLI output testing - see MODE5_COUNTTEXT)

# - MODE=6 [ALPHA]: Multi threaded (ThreadSync) Valgrind output testing (set TEXT)

# - MODE=7 [ALPHA]: Multi threaded (ThreadSync) mysql CLI/pquery client output testing (set TEXT)

# - MODE=8 [ALPHA]: Multi threaded (ThreadSync) mysqld error output log testing (set TEXT)

# - MODE=9 [ALPHA]: Multi threaded (ThreadSync) crash testing

# - SKIPSTAGEBELOW: Stages up to and including this one are skipped (default=0).

# - SKIPSTAGEABOVE: Stages above and including this one are skipped (default=9).

# - TEXT: Text to look for in MODEs 1,2,3,5,6,7,8. Ignored in MODEs 4 and 9.

# Can contain extended grep (i.e. grep -E --binary-files=text or egrep)+regex syntax like "^ERROR|some_other_string". Remember this is regex: specify | as \| etc.

# For MODE5, you would use a mysql CLI to get the desired output "string" (see example given above) and then set MODE5_COUNTTEXT

# - PQUERY_MOD: 1: use pquery, 0: use mysql CLI. Causes reducer.sh to use pquery instead of the mysql client for replays (default=0). Supported for MODE=1,3,4

# - PQUERY_LOC: Location of the pquery binary (retrieve pquery like this; $ cd ~; bzr branch lp:percona-qa; # then ref ~/percona-qa/pquery/pquery[-ms])

# - PQUERY_EXTRA_OPTIONS: Extra options to pquery which will be added to the pquery command line. This is used for query correctness trials

# - PXC_MOD: 1: bring up 3 node Percona XtraDB Cluster instead of default server, 0: use default non-cluster server (mysqld)

# - ENCRYPTION_RUN: 1: Start PXC with encryption options, 0: Start PXC without encryption options

# - GRP_RPL_MOD: 1: bring up 3 node Group Replication instead of default server, 0: use default non-cluster server (mysqld)

# see lp:/percona-qa/pxc-pquery/new/pxc-pquery_info.txt and lp:/percona-qa/docker_info.txt for more information on this. See above for some limitations etc.

# IMPORTANT NOTE: If this is set to 1, ftm, these settings (and limitations) are automatically set: INHERENT: PQUERY_MOD=1, LIMTATIONS: FORCE_SPORADIC=0,

# SPORADIC=0, FORCE_SKIPV=0, SKIPV=1, MYEXTRA="", MULTI_THREADS=0

# - PXC_ISSUE_NODE: This indicates which node you would like to be checked for presence of the issue. 0 = Any node. Valid options: 0, 1, 2, or 3. Only works

# for MODE=4 currently.

# - GRP_RPL_ISSUE_NODE: This indicates which node you would like to be checked for presence of the issue. 0 = Any node. Valid options: 0, 1, 2, or 3. Only works

# for MODE=4 currently.

# - PXC_DOCKER_COMPOSE_LOC: Location of the Docker Compose file used to bring up 3 node Percona XtraDB Cluster (using images previously prepared by "new" method)

# - QUERYTIMEOUT: Number of seconds to wait before terminating a query (similar to RQG's querytimeout option). Do not set < 40 to avoid initial DDL failure

# Warning: do not set this smaller then 1.5x what was used in RQG. If set smaller, the bug may not reproduce. 1.5x instead of 1x is a simple precaution

# - TS_TRXS_SETS [ALPHA]: For ThreadSync simplification (MODE 6+), use the last x set of thread actions only

# (i.e. the likely crashing statements are likely at the end only) (default=1, 0=disable)

# Increase to increase reproducibility, but increasing this exponentially also slightly lowers reliability. (DEBUG_SYNC vs session sync issues)

# - TS_DBG_CLI_OUTPUT: ONLY activate for debugging. We need top speed for the mysql CLI to reproduce multi-threaded issues accurately

# This turns on -vvv debug output for the mysql client (Best left disabled=default=0)

# Turning this on *will* significantly reduce (if not completely nullify) issue reproducibility due to excessive disk logging

# - TS_DS_TIMEOUT: Number of seconds to wait in a DEBUG_SYNC lock situation before terminating current DEBUG_SYNC lock holds

# - TS_VARIABILITY_SLEEP: Number of seconds to wait before a new transaction set is processed (may slightly increase/decrease issue reproducibility)

# Suggested values: 0 (=default) or 1. This is one of the first parameters to test (change from 0 to 1) if a ThreadSync issue is not reproducible

# - WORKDIR_LOCATION: Select which medium to use to store the working directory (Note that some issues require the extra speed of setting 1,2 or 3 to reproduce)

# (Note that the working directory is also copied to /tmp/ after the reducer run finishes if tmpfs or ramfs are used)

# - WORKDIR_LOCATION=0: use /tmp/ (disk bound)

# - WORKDIR_LOCATION=1: use tmpfs (default)

# - WORKDIR_LOCATION=2: use ramfs (setup: sudo mkdir -p /mnt/ram; sudo mount -t ramfs -o size=4g ramfs /mnt/ram; sudo chmod -R 777 /mnt/ram;)

# - WORKDIR_LOCATION=3: use a specific storage device (like an ssd or other [fast] storage device), mounted as WORKDIR_M3_DIRECTORY

# - WORKDIR_M3_DIRECTORY: If WORKDIR_LOCATION is set to 3, then this directory is used

# - STAGE1_LINES: When the testcase becomes smaller than this number of lines, proceed to STAGE2 (default=90)

# Only change if reducer keeps trying to reduce by 1 line in STAGE1 for a long time (seen very rarely)

# - MYEXTRA: Extra options to pass to myqsld (for instance "--core" is handy in some cases, for instance with highly sporadic issues to capture a core)

# (Btw, --core should generally be left disabled to obtain cleaner output ("core dumped" stdout messages, less space used, faster reducer runs)

# - Also, --no-defaults as set in the default is removed automatically later on. It is just present here to highlight it's effectively (seperately) set.

# - BASEDIR: Full path to MySQL basedir (example: "/mysql/mysql-5.6").

# If the directory name starts with '/mysql/' then this may be ommited (example: BASEDIR="mysql-5.6-trunk")

# - MULTI_THREADS: This option was an internal one only before. Set it to change the number of threads Reducer uses for the verify stage intially, and for

# reduction of sproradic issues if the verify stage found it is a sporadic issue. Recommended: 10, based on experience/testing/time-proven correctness.

# Do not change unless you need to. Where this may come in handy, for a single occassion, is when an issue is hard to reproduce and very sporadic. In this

# case you could activate FORCE_SKIPV (and thus automatically also FORCE_SPORADIC) which would skip the verify stage, and set this to a higher number for

# example 20 or 30. This would then immediately boot into 20 or 30 threads trying to reduce the issue with subreducers (note: thus 20 or 30x mysqld...)

# A setting less then 10 is really not recommended as a start since sporadic issues regularly only crash a few threads in 10 or 20 run threads.

# - MULTI_THREADS_INCREASE: this option configures how many threads are added to MULTI_THREADS if the original MULTI_THREADS setting did not prove to be

# sufficient to trigger a (now declared highly-) sporadic issue. Recommended is setting 5 or 10. Note that reducer has a limit of MULTI_THREADS_MAX (50)

# threads (this literally means 50x mysqld + client thread(s)) as most systems (including high-end servers) start to seriously fail at this level (and

# earlier) Example; if you set MULTI_THREADS to 10 and MULTI_THREADS_INCREASE to 10, then the sequence (if no single reproduce can be established) will be:

# 10->20->30->40->50->Issue declared non-reproducible and program end. By this stage, the testcase has executed 6 verify levels *(10+20+30+40+50)=900 times.

# Still, even in this case there are methods that can be employed to let the testcase reproduce. For further ideas what to do in these cases, see;

# http://bazaar.launchpad.net/~percona-core/percona-qa/trunk/view/head:/reproducing_and_simplification.txt

# - FORCE_SPORADIC=0 or 1: If set to 1, STAGE1_LINES setting is ignored and set to 3, unless it was set to a non-default number (i.e. !=90 - to enable

# reduction of issues via MULTI until a given amount of lines is reached, which is handy for tools like pquery-reach.sh where a mix of sporadic and

# non-sporadic issues may be seen). MULTI reducer mode is used after verify, even if issue is found to seemingly not be sporadic (i.e. all verify

# threads reproduced the issue). This can be handy for issues which are very slow to reduce or which, on visual inspection of the testcase reduction

# process are clearly sporadic (i.e. it comes to 2 line chunks with still thousands of lines in the testcase and/or there are many trials without the

# issue being observed. Another situation which would call for use of this parameter is when produced testcases are still greater then 15 to 80 lines -

# this also indicates a possibly sporadic issue (even if verify stage manages to produce it against all started subreducer threads).

# Note that this may be a bug in reducer too - i.e. a mismatch between verify stage and stage 1. Yet, if that were true, the issue would likely not

# reproduce to start with. Another plausible reason for this occurence (all threads verified in verify stage but low frequency reproduction later on) is

# the existence of all threads in verify stage vs 1 thread in stage 1. It has been observed that a very loaded server (or using Valgrind as it also slows the

# code down significantly) is better at reproducing (many) issues then a low-load/single-thread-running machine. Whatever the case, this option will help.

# - FORCE_SKIV=0 or 1: If set to 1, FORCE_SPORADIC is automatically set to 1 also. This option skips the verify stage and goes straight into testcase reduction

# mode. Ideal for issues that have a very low reproducibility, at least initially (usually either increases or decreases during a simplification run.)

# Note that skipping the verify stage means that you may not be sure if the issue is reproducibile untill it actually reproduces (how long is a piece of

# string), and the other caveat is that the verify stage normally does some very important inital simplifications which is now skipped. It is suggested that

# if the issue becomes more reproducible during simplification, to restart reducer with this option turned off. This way you get the best of both worlds.

# - PQUERY_MULTI=0 or 1: If set to 1, FORCE_SKIV (and thus FORCE_SPORADIC) are automatically set to 1 also. This is true multi-threaded testcase reduction,

# and it is based on random replay. Likely this will be slow, but effective. Alpha quality. This option removes the --no-shuffle option for pquery (i.e.

# random replay) and sets pquery options --threads=x (x=PQUERY_MULTI_CLIENT_THREADS) and --queries=5*testcase size. It also sets the number of subreducer

# threads to PQUERY_MULTI_THREADS. To track success/status, view reducer output and/or check error logs;

# $ grep -E --binary-files=text "Assertion failure" /dev/shm/{reducer's epoch}/subreducer/*/error.log

# Note that, idem to when you use FORCE_SKIV and/or FORCE_SPORADIC, STAGE1_LINES is set to 3. Thus, reducer will likely never completely "finish" (3 line

# testases are somewhat rare), as it tries to continue to reduce the test to 3 lines. Just watch the output (reducer continually reports on remaining number

# of lines and/or filesize) and decide when you are happy with the lenght of any reduced testcase. Suggested for developer convenience; 5-10 lines or less.

# - PQUERY_MULTI_THREADS: Think of this variable as "the initial setting for MULTI_THREADS" when PQUERY_MULTI mode is enabled; the initial number of subreducers

# - PQUERY_MULTI_CLIENT_THREADS: The number of client threads used for PQUERY_MULTI (see above) replays (i.e. --threads=x for pquery)

# - PQUERY_MULTI_QUERIES: The number of queries to execute for each and every trial before pquery ends (unless the server crashes/asserts). Must be

# sufficiently high, given that the random replay which PQUERY_MULTI employs may not easily trigger an issue (and especially not if also sporadic)

# - PQUERY_REVERSE_NOSHUFFLE_OPT=0 or 1: If set to 1, PQUERY_MULTI runs will use --no-shuffle (the reverse of normal operation), and standard pquery (not multi-

# threaded) will use shuffle (again the reverse of normal operation). This is a very handy option to increase testcase reproducibility. For example, when

# reducing a non-multithreaded testcase (i.e. normally --no-shuffle would be in use), and reducer.sh gets 'stuck' at around 60 lines, setting this to

# on will start replaying the testcase randomly (shuffled). This may increase reproducibility. The final run scripts will have matching --no-shuffle or

# shuffle (i.e. no --no-shuffle present) set. Note that this may mean that a testcase has to be executed a few or more times given that if shuffle is

# active (pquery's default, i.e. no --no-shuffle present), the testcase may replay differently then to what is needed. Powerful option, slightly confusing.

# - TIMEOUT_COMMAND: this can be used to set a timeout command for mysqld. It is prefixed to the mysqld startup. This is handy when encountering a shutdown

# or server hang issue. When the timeout is reached, mysqld is terminated, but reduction otherwise happens as normal. Note that reducer will need some way

# to establish that an actual problem was triggered. For example, suppose that a shutdown issue shows itself in the error log by starting to output INNODB

# STATUS MONITOR output whenever the shutdown issue is occuring (i.e. server refuses to shutdown and INNODB STATUS MONITOR output keeps looping & end of

# the SQL input file is apparently never reached). In this case, after a timeout of x minutes, thanks to the TIMEOUT_COMMAND, mysqld is terminated. After

# the termination, reducer checks for "INNODB MONITOR OUTPUT" (MODE=3). It sees or not sees this output, and hereby it can continue to reduce the testcase

# further. This would have been using MODE=3 (check error log output). Another method may be to interleave the SQL with a SHOW PROCESSLIST; and then

# check the client output (MODE=2) for (for example) a runaway query. Different are issues where there is a 1) complete hang or 2) an issue that does not

# or cannot!) represent itself in the error log/client log etc. In such cases, use TIMEOUT_CHECK and MODE=0.

# - TIMEOUT_CHEK: used when MODE=0. Though there is no connection with TIMEOUT_COMMAND, the idea is similar; When MODE=0 is active, a timeout command prefix

# for mysqld is auto-generated by reducer.sh. Note that MODE=0 does NOT check for specific TEXT string issues. It just checks if a timeout was reached

# at the end of each trial run. Thus, if a server was hanging, or a statement ran for a very long time (if not terminated by the QUERYTIMEOUT setting), or

# a shutdon was initiated but never completed etc. then reducer.sh will notice that the timeout was reached, and thus assume the issue reproduced. Always

# set this setting at least to 2x the expected testcase run/duration lenght in seconds + 30 seconds extra. This longer duration is to prevent false

# positives. Reducer auto-sets this value as the timeout for mysqld, and checks if the termination of mysqld was within 30 seconds of this duration.

# - FORCE_KILL=0 or 1: If set to 1, then reducer.sh will forcefully terminate mysqld instead of using mysqladmin. This can be used when for example

# authentication issues prevent mysqladmin from shutting down the server cleanly. Normally it is recommended to leave this =0 as certain issues only

# present themselves at the time of mysqld shutdown. However, in specific use cases it may be handy. Not often used. Auto-disabled for MODE=0.

# ======== Gotcha's

# - When reducing an SQL file using for example FORCE_SKIPV=1, FORCE_SPORADIC=1, PQUERY_MULTI=0, PQUERY_REVERSE_NOSHUFFLE_OPT=1, PQUERY_MOD=1, then reducer

# will replay the SQL file, using pquery (PQUERY_MOD=1), using a single client (i.e. pquery) thread against mysqld (PQUERY_MULTI=0), in a sql shuffled order

# (PQUERY_REVERSE_NOSHUFFLE_OPT=1) untill (FORCE_SKIPV=1 and FORCE_SPORADIC=1) it hits a bug. But notice that when the partially reduced file is written

# as _out, it is normally not valid to re-start reducer using this _out file (for further reduction) using PQUERY_REVERSE_NOSHUFFLE_OPT=0. The reason is

# that the sql replay order was random, but _out is generated based on the original testcase (sequential). Thus, the _out, when replayed sequentially,

# may not re-hit the same issue. Especially when things are really sporadic this can mean having to wait long and be confused about the results. Thus,

# if you start off with a random replay, finish with a random replay, and let the final bug testcase (auto-generated as {epoch}.*) be random replay too!

# ======== General develoment information

# - Subreducer(s): these are multi-threaded runs of reducer.sh started from within reducer.sh. They have a specific role, similar to the main reducer.

# At the moment there are only two such specific roles: verfication (reproducible yes/no + sporadic yes/no) and simplification (terminate a subreducer batch

# (all of it) once a simpler testcase is found by one of the subthreads (subreducers), and use that testcase to again start new simplification subreducers.)

# - The files that are initially seen in the root working directory (i.e. $WORKD) are those generated by the step "[Init] Setting up standard working template",

# they are not an actual replay of any SQL file, at least not intitially; once the processing continues past this initial template creation, then the results

# (i.e. the results of actual replays of SQL) will be in either;

# - The subreducer directories $WORKD/subreducer/<nr>/ (ref above), provided reducer.sh is working in MULTI mode (even the standard VERIFY stage is [MULTI])

# - Or, they will be in the same aforementioned directory $WORKD (and the output files from the initial template creation will now have been overwritten,

# though not the actual template), provided reducer.sh is working in single-threaded reduction mode (i.e. [MULTI] mode is not active).

# - Never use grep, always use egrep. See the next line why. Remember also that [0-9]\+ (a regex valid for grep) is written as [0-9]+ when using egrep/grep -E --binary-files=text.

# grep -E --binary-files=text is the same as egrep. It is best to use grep -E --binary-files=text because egrep will likely be deprecated from various OS'es at some point.

# - When using grep -E --binary-files=text, ALWAYS use --binary-files=text to avoid issues with hex characters causing non-reproducibility and/or grep playing up. If you see

# things like 'Binary file ... matches' as grep output it means you have executed grep against a file with binary chars, which is seen by the system as a

# binary file (even though it may be a flat sql text file with a few hex characters in it). Adding the --binary-files=text will correctly process the file.

# ======== Ideas for improvement

# - The write of the file should be atomic - i.e. if reducer is interrupted during a testcase_out write, the file may be faulty. Check if this is so & fix

# - STAGE8 does currently know/consider whetter an issue is sporadic (alike to other STAGES, except STAGE 1). We could have an additional option like

# STAGE8_FORCE_SPORADIC=0/1 Which would - specifically for STAGE 8 - try each option x times (STAGE8_SPORADIC_ATTEMPTS) when an issue is found to be (by the

# auto-sporadic issue detection) or is forced to be (by STAGE8_FORCE_SPORADIC=1) sporadic. This allows easier reduction of mysqld options for sporadic issues)

# - A new mode could do this; main thread (single): run SQL, secondary thread (new functionality): check SHOW PROCESSLIST for certain regex TEXT regularly. This

# would allow creating testcases for queries that have a long runtime. This new functionality likely will live outside process_outcome() as it is a live check

# - Incorporate 3+ different playback options: SOURCE ..., redirection with <, redirection with cat, (stretch goal; replay via MTR), etc. (there may be more)

# THIS FUNCTIONALITY WAS ADDED 09-06-2016. "An expansion of this..." below is not implmeneted yet

# - It has been clearly shown that different ways of replaying SQL may trigger a bug where other replay options do not. This looks to be more related to for

# example timing/server access method then to an inherent/underlying bug in for example the mysql client (CLI) workings. As such, the "resolution" is not

# to change ("fix") the client instead exploit this difference between replay options to trigger/reproduce bugs/replay test cases in multiple ways.

# - An expansion of this could be where the initial stage (as it goes through it's iterations) replays each next iteration with a different replay method.

# This is not 100% covering however, as the last stage (with the least amount of changes to the SQL input file) would replay with replay method/option x,

# while x may not be the replay option which triggers the bug at hand. As such, a few more verify stage rounds (there's 6 atm - each with 10 replay threads)

# may be needed to replay (partly "again", but this time with the least changed SQL file) the same SQL with each replay option. This would thus result in

# reducer needing a bit more time to do the VERIFY stage, but likely with good improved bug reproducibility. Untill this functionality is implemented,

# see the following file/page for reproducing & simplification ideas, which (if all followed diligently) usually result in bugs becoming reproducible;

# http://bazaar.launchpad.net/~percona-core/percona-qa/trunk/view/head:/reproducing_and_simplification.txt

# - PXC Node work: rm -Rf's in other places (non-supported subreducers for example) will need sudo. Also test for sudo working correctly upfront

# - Add a MYEXRA simplificator at end (extra stage) so that mysqld options are minimal

# - Improve ";" work in STAGE4 (";" sometimes missing from results - does not affect reproducibility)

# - Improve VALGRIND/ERRORLOG run work (complete?)

# - Improve clause elimination when sub queries are used: "ORDER BY f1);" is not filtered due to the ending ")"

# - Keep 'success counters' over time of regex replacements so that reducer can eliminate those that are not effective

# Do this by proceduralizing the sed and then writing the regexes to a file with their success/failure rates

# - Include a note for Valgrind runs on a "universal" string - a string which would be found if there were any valgrind erros

# Something like "[1-9]* ERRORS" or something

# - Keep counters over time of which sed's have been successfull or not. If after many different runs, a sed remains 0 success, remove it

# - Proceduralize stages and re-run STAGE2 after the last stage as this is often beneficial for # of lines (and remove last [Info] line in [Finish])

# - Have to find some solution for the crash in tzinfo where reducer needs to use a non-Valgrind-instrumented build for tzinfo

# - (process) script could pass all RQG-set extra mysqld options into MYEXTRA or another variable to get non-reproducible issues to work

# - STAGE6: can be improved slightly furhter. See function for ideas.

# Also, the removal of a column fails when a CREATE TABLE statement includes KEY(col), so maybe these keys can be pre-dropped or at the same time

# Also, try and swap any use of the column to be removed to the name of column-1 (just store it in a variable) to avoid column missing error

# And at the same time still promote removal of the said column

# - start_mysqld_main() needs a bit more work to generate the $WORK_RUN file for MODE6+ as well (multiple $WORKO files are used in MODE6+)

# Also remove the ifthen in finish() which test for MODE6+ for this and reports that implementation is not complete yet

# - STAGE6: 2 small bugs (ref output lines below): 1) Trying to eliminate a col that is not one & 2) `table0_myisam` instead of table0_myisam

# Note that 2) may not actually be a bug; if the simplifacation of "'" failed for a sporadic testcase (as is the case here), it's hard to fix (check)

# | 2013-08-19 10:35:04 [*] [Stage 6] [Trial 2] [Column 22/22] Trying to eliminate column '/*Indices*/' in table '`table0_myisam`'

# | sed: -e expression #1, char 41: unknown command: `*'

# - Need another MODE which will look for *any* Valgrind issue based on the error count not being 0 (instead of named MODE1)

# Make a note that this may cause issues to be missed: often, after simplification, less Valgrind errors are seen as the entire

# SQL trace likely contained a number of issues, each originating from different Valgrind statements (can multi-issue be automated?)

# - Need another MODE which will attempt to crash the server using the crashing statement from the log, directly starting the vardir

# left by RQG. If this works, dump the data, add crashing statement and load in a fresh instance and re-try. If this works, simplify.

# - "Previous good testcase backed up as $WORKO.prev" was only implemented for 1) parent seeing a new simplification subreducer testcase and

# 2) main single-threaded reducer seeing a new testcase. It still needs to be added to multi-threaded (ThreadSync) (i.e. MODE6+) simplification. (minor)

# - Multi-threaded simplification: thread-elimination > DATA + SQL threads simplified as if "one file" but accross files.

# Hence, all stages need to be updated to be multi-threaded/TS aware. Fair amount of work, but doable.

# See initial section of 'Verify' for some more information around multi_reducer_decide_input

# - Multi-threaded simplification: # threads left + non-sporadic: attempt all DATA+SQL1+SQLx combinations. Then normal simplify.

# Sporadic: normal simplify immediately.

# - Multi-threaded simplification of sporadic issues: could also start # subreducer sessions and have main reducer watch for _out creation.

# Once found, abort all live subreducer threads and re-init with found _out file. Maybe a safety copy of original file should be used for running.

# - MODE9 work left

# - When 2 threads are left (D+2T) then try MODE4 immediately instead of executing x TS_TE_ATTEMPTS attempts

# - In single thread replay it should always do grep -E --binary-files=text -v "DEBUG_SYNC" as DEBUG_SYNC does not make sense there (cosmetic, would be filtered anyway)

# - bash$ echo -ne "test\r"; echo "te2" > use this implementation for same-line writing of threa fork commands etc

# - TS_TRXS_SETS "greps" not fully corret yet: setting this to 10 lead to 2x main delay while it should have been 10. Works correctly when "1"

# - TS_TRXS_SETS processing can be automated - and this is the simplification: test last, test last+1, test last+2, untill crash. (or chuncks?)

# - Check if it is a debug server by issuing dummy DEBUG_SYNC command and see if it waits (TIMEOUT?)

# - cut_threadsync_chunk is not in use at the moment, this will be used? but try ts_thread_elimination first

# - Need to capture interrupt (CTRL+C) signal and do some end-processing (show info + locations + copy to tmp if tmpfs/ramfs used)

# - If "sed: -e expression #1, char 44: unknown option to `s'" text or similar is seen in the output, it is likely due to the #VARMOD# block

# replacement in multi_reducer() failing somewhere. Update RV 16/9: Added functionality to fix/change ":" to "\:" ($FIXED_TEXT) to avoid this error.

# - Implement cmd line options instead of in-file options. Example:

# while [ "$1" != ""]; do

# case $1 in

# -m | --mode shift;MODE=$1;; # shift to get actual file name into $1

# -f | --file shift;file=$1;;

# *) no_options;exit 1;;

# esac

# shift

# done

# - Optimization: let 'Waiting for any forked subreducer threads to find a shorter file (Issue is sporadic: this will take time)' work for 30 minutes

# or so, depending on file size. If no issue is found by then, restart or increase number of threads by 5.

# ======== Internal variable Reference

# $WORKD = Working directory (i.e. likely /tmp/<epoch>/ or /dev/shm/<epoch>)

# $INPUTFILE = The original input file (the file to reduce). This file, and this variable, are never changed (to protect the original file from being changed).

# $WORK_BUG_DIR = The directory in which the original input file resides (i.e. $INPUTFILE, which may have been set to $1 specifically as well). In this directory the output files will be stored

# $WORKF = This is *originally* a copy of $INPUTFILE, seen in the working directory as $WORKD/in.sql

# work From it are then made chunk deletes etc. and the result is stored in the $WORKT file. Then, $WORKT ovewrites $WORKF when

# file a [for MODE4+9: "likely the same", for other MODES: "the same"] issue was located when executing $WORKT

# $WORKT = A temporary "made smaller" (and thus changed) version of $WORKF, seen in the working directory as $WORKD/in.tmp

# temp It may or may not cause the same issue like $WORKF can. This file is overwritten each time a new "to be tested" version is being created

# file $WORKT overwrites $WORKF and $WORKO when a [for MODE4+9: "likely the same", for other MODES: "the same"] issue was located when executing $WORKT

# $WORKO = The reduced version of $WORKF, stored in the same directory of the original input file as <name>_out

# outf This file definitely causes the same issue as $INPUTFILE can, while being smaller

# $WORK_INIT, $WORK_START, $WORK_STOP, $WORK_CL, $WORK_RUN, $WORK_RUN_PQUERY: Vars that point to various start/run scripts that get added to testcase working dir

# $WORK_OUT: an eventual copy of $WORKO, made for the sole purpose of being used in combination with $WORK_RUN etc. This makes it handy to bundle them as all

# of them use ${EPOCH} in the filename, so you get {some_epochnr}_start/_stop/_cl/_run/_run_pquery/.sql

# Set ASAN coredump options

# https://github.com/google/sanitizers/wiki/SanitizerCommonFlags

# https://github.com/google/sanitizers/wiki/AddressSanitizerFlags

export ASAN_OPTIONS=quarantine_size_mb=512:atexit=true:detect_invalid_pointer_pairs=1:dump_instruction_bytes=true:abort_on_error=1

# ===== [SPECIAL MYEXTRA SECTION START] Preparation for STAGES 8 and 9: special MYEXTRA startup option sets handling

# Important: If you add a section below for additional startup option sets, be sure to add the final outcome to SPECIAL_MYEXTRA_OPTIONS at the end of this section (marked by "[SPECIAL MYEXTRA SECTION END]")

# And additionaly to add a new trial in STAGE9 to cover the additional startup option set created here.

SPECIAL_MYEXTRA_OPTIONS=

# === Check TokuDB & RocksDB storage engine options, .so availability, split options into ROCKSDB and TOKUDB variables, and cleanup MYEXTRA to remove the related options

# SE Removal approach; 1) If the engine is referred to by .so reference in MYEXTRA, reducer.sh uses it, but reducer.sh ensure the engine .so file exists

# 2) Any reference to the engine is removed from MYEXTRA and stored in two variables TOKUDB/ROCKSDB to allow more control/testcase reducability

# 3) Testcase reduction removal of engines (one-by-one) is tested in STAGE9

MYSQL_VERSION=$(${BASEDIR}/bin/mysqld --version 2>&1 | grep -oe '[0-9]\.[0-9][\.0-9]*' | head -n1)

#Format version string (thanks to wsrep_sst_xtrabackup-v2)

normalize_version(){

local major=0

local minor=0

local patch=0

# Only parses purely numeric version numbers, 1.2.3

# Everything after the first three values are ignored

if [[ $1 =~ ^([0-9]+)\.([0-9]+)\.?([0-9]*)([\.0-9])*$ ]]; then

major=${BASH_REMATCH[1]}

minor=${BASH_REMATCH[2]}

patch=${BASH_REMATCH[3]}

fi

printf %02d%02d%02d $major $minor $patch

}

#Version comparison script (thanks to wsrep_sst_xtrabackup-v2)

check_for_version()

{

local local_version_str="$( normalize_version $1 )"

local required_version_str="$( normalize_version $2 )"

if [[ "$local_version_str" < "$required_version_str" ]]; then

return 1

else

return 0

fi

}

TOKUDB=

ROCKSDB=

if [[ "${MYEXTRA}" == *"ha_rocksdb.so"* ]]; then

if [ -r ${BASEDIR}/lib/mysql/plugin/ha_rocksdb.so ]; then

ROCKSDB="$(echo "${MYEXTRA}" | grep -o "\-\-plugin[-_][^ ]\+ha_rocksdb.so" | head -n1)" # Grep all text including and after ' --plugin[-_]' (upto any space as a new option starts there) upto and including the last 'ha_rocksdb.so' for that option

MYEXTRA="$(echo "${MYEXTRA}" | sed "s|${ROCKSDB}||g")"

# The below issues should never happen in the Percona pquery framework as we simply use;

# --plugin-load-add=tokudb=ha_tokudb.so --tokudb-check-jemalloc=0 --plugin-load-add=rocksdb=ha_rocksdb.so --init-file=/home/roel/percona-qa/plugins_57.sql

# And the init-file loads any other required plugins using the same .so file. These options (in MYEXTRA) are not complex and easy too parse as per below -

# and this is handled fine by the code here. It would only happen if someone used a complex string like the one shown in https://jira.percona.com/browse/DOC-444

if [[ "${MYEXTRA}" == *"ha_rocksdb.so"* ]]; then

echo "Error: The MYEXTRA string is formulated in a seemingly complex manner; it should contain (per engine) only one '--plugin-load[-add]=...ha_....so' (and note that --plugin-load can only be used once; perhaps best to use --plugin-load-add for each engine)."

echo "Please simplify it, or improve the code in reducer.sh which handles this (search for this text)."

echo "Terminating now."

exit 1

elif [[ "${ROCKSDB}" == *"ha_tokudb.so"* ]]; then

echo "Error: The MYEXTRA string is formulated in a seemingly complex manner; it should contain (per engine) only one '--plugin-load[-add]=...ha_....so' (and note that --plugin-load can only be used once; perhaps best to use --plugin-load-add for each engine)."

echo "It looks like the ha_tokudb.so plugin load call was nested inside the --plugin-load[-add]=...ha_rocksdb.so plugin load call."

echo "Please simplify it by using a separate --plugin-load-add for each engine, or improve the code in reducer.sh which handles this (search for this text) to extract the TokuDB load code into the TOKUDB variable at this point in the code (complex)."

echo "Terminating now."

exit 1

fi

else

echo "Error: MYEXTRA contains ha_rocksdb.so, yet ${BASEDIR}/lib/mysql/plugin/ha_rocksdb.so des not exist."

echo "Terminating now."

exit 1

fi

fi

if [[ "${MYEXTRA}" == *"ha_tokudb.so"* ]]; then

if [ -r ${BASEDIR}/lib/mysql/plugin/ha_tokudb.so ]; then

TOKUDB="$(echo "${MYEXTRA}" | grep -o "\-\-plugin[-_][^ ]\+ha_tokudb.so" | head -n1)" # Grep all text including and after '--plugin[-_]' (upto any space as a new option starts there) upto and including the last 'ha_tokudb.so' for that option

MYEXTRA="$(echo "${MYEXTRA}" | sed "s|${TOKUDB}||g")"

if [[ "${MYEXTRA}" == *"--tokudb"[-_]"check"[-_]"jemalloc"* ]]; then

TOKUDBJC="$(echo "${MYEXTRA}" | grep -o "\-\-tokudb[-_]check[-_]jemalloc[^ ]*" | head -n1)" # Grep all text including and after '--tokudb[-_]check[-_]jemalloc' upto the first space

MYEXTRA="$(echo "${MYEXTRA}" | sed "s|${TOKUDBJC}||g")"

TOKUDB="$(echo "${TOKUDB} ${TOKUDBCJ}")"

fi

# The below issues should never happen in the Percona pquery framework; ref info above in ha_rocksdb.so section

if [[ "${MYEXTRA}" == *"ha_tokudb.so"* ]]; then

echo "Error: The MYEXTRA string is formulated in a seemingly complex manner; it should contain (per engine) only one '--plugin-load[-add]=...ha_....so' (and note that --plugin-load can only be used once; perhaps best to use --plugin-load-add for each engine)."

echo "Please simplify it, or improve the code in reducer.sh which handles this (search for this text)."

echo "Terminating now."

exit 1

elif [[ "${TOKUDB}" == *"ha_rocksdb.so"* ]]; then

echo "Error: The MYEXTRA string is formulated in a seemingly complex manner; it should contain (per engine) only one '--plugin-load[-add]=...ha_....so' (and note that --plugin-load can only be used once; perhaps best to use --plugin-load-add for each engine)."

echo "It looks like the ha_rocksdb.so plugin load call was nested inside the --plugin-load[-add]=...ha_tokudb.so plugin load call."

echo "Please simplify it by using a separate --plugin-load-add for each engine, or improve the code in reducer.sh which handles this (search for this text) to extract the RocksDB load code into the ROCKSDB variable at this point in the code (complex)."

echo "Terminating now."

exit 1

fi

else

echo "Error: MYEXTRA contains ha_tokudb.so, yet ${BASEDIR}/lib/mysql/plugin/ha_tokudb.so des not exist."

echo "Terminating now."

exit 1

fi

fi

if [[ "${MYEXTRA}" == *"--tokudb"[-_]"check"[-_]"jemalloc"* ]]; then

echo "Error: MYEXTRA contains --tokudb-check-jemalloc, yet ha_tokudb.so is not present in the MYEXTRA string."

echo "Terminating now."

exit 1

fi

# === Check binary log encryption options, split it into a BL_ENCRYPTION variable, and cleanup MYEXTRA to remove the related options

BL_ENCRYPTION=

if [[ "${MYEXTRA}" == *"encrypt"[-_]"binlog"* ]]; then

if [[ ! "${MYEXTRA}" == *"master"[-_]"verify"[-_]"checksum"* ]]; then

echo "Error: --encrypt-binlog is present in MYEXTRA whereas --master-verify-checksum is not (as required by binary log encryption). Please fix this."

echo "Terminating now."

exit 1

fi

if [[ ! "${MYEXTRA}" == *"binlog"[-_]"checksum"* ]]; then

echo "Error: --encrypt-binlog is present in MYEXTRA whereas --binlog-checksum is not (as required by binary log encryption). Please fix this."

echo "Terminating now."

exit 1

fi

BL_ENCRYPTION="$(echo "${MYEXTRA}" | grep -o "\-\-encrypt[-_]binlog[^ ]*" | head -n1)" # Grep all text including and after '--encrypt_binlog' upto the first space

MYEXTRA="$(echo "${MYEXTRA}" | sed "s|${BL_ENCRYPTION}||g")"

BL_ENCRYPTIONMVC="$(echo "${MYEXTRA}" | grep -o "\-\-master[-_]verify[-_]checksum[^ ]*" | head -n1)" # Grep all text including and after '--master[-_]verify[-_]checksum' upto the first space

MYEXTRA="$(echo "${MYEXTRA}" | sed "s|${BL_ENCRYPTIONMVC}||g")"

BL_ENCRYPTIONBC="$(echo "${MYEXTRA}" | grep -o "\-\-binlog[-_]checksum[^ ]*" | head -n1)" # Grep all text including and after '--binlog[-_]checksum' upto the first space

MYEXTRA="$(echo "${MYEXTRA}" | sed "s|${BL_ENCRYPTIONBC}||g")"

BL_ENCRYPTION="$(echo "${BL_ENCRYPTION} ${BL_ENCRYPTIONMVC} ${BL_ENCRYPTIONBC}")"

fi

# === Check keyring file encryption options, split it into a KF_ENCRYPTION variable, and cleanup MYEXTRA to remove the related options

KF_ENCRYPTION=

if [[ "${MYEXTRA}" == *"plugin"[-_]"load=keyring_file.so"* ]]; then

if [[ ! "${MYEXTRA}" == *"keyring"[-_]"file"[-_]"data"* ]]; then

echo "Error: --[early-]plugin-load=keyring_file.so is present in MYEXTRA whereas --keyring_file_data (as required by the keyring file plugin) is not. Please fix this."

echo "Terminating now."

exit 1

fi

KF_ENCRYPTION="$(echo "${MYEXTRA}" | grep -o "\-\-[^ ]\+keyring_file.so" | head -n1)" # Grep all text (which is not a space) including and after '--' upto and including 'keyring_file.so'

MYEXTRA="$(echo "${MYEXTRA}" | sed "s|${KF_ENCRYPTION}||g")"

if [[ "${MYEXTRA}" == *"--keyring"[-_]"file"[-_]"data"* ]]; then

KF_ENCRYPTIONFD="$(echo "${MYEXTRA}" | grep -o "\-\-keyring[-_]file[-_]data[^ ]*" | head -n1)" # Grep all text including and after '--keyring[-_]file[-_]data' upto the first space

MYEXTRA="$(echo "${MYEXTRA}" | sed "s|${KF_ENCRYPTIONFD}||g")"

KF_ENCRYPTION="$(echo "${KF_ENCRYPTION} ${KF_ENCRYPTIONFD}")"

fi

else

if [[ "${MYEXTRA}" == *"keyring"[-_]"file"[-_]"data"* ]]; then

echo "Error: --keyring_file_data is present in MYEXTRA whereas --[early-]plugin-load=keyring_file.so is not. Please fix this."

echo "Terminating now."

exit 1

fi

fi

# === Check Binary logging options, split it into a BINLOG variable, and cleanup MYEXTRA to remove the related options

BINLOG=

if [[ "${MYEXTRA}" == *"server"[-_]"id"* ]]; then

if [[ ! "${MYEXTRA}" == *"log"[-_]"bin"* ]]; then

if [[ ! "$(${BASEDIR}/bin/mysqld --version | grep -E --binary-files=text -oe '5\.[1567]|8\.[0-9]' | head -n1)" =~ ^8.[0-9]$ ]]; then # version is not 8.0 (--log-bin is not required as it is default already (8.0 has binary logging enabled by default))

echo "Error: --server-id is present in MYEXTRA whereas --log-bin is not. Please fix this."

echo "Terminating now."

exit 1

else

echo "Warning: --server-id is present in MYEXTRA whereas --log-bin is not. This is a valid setup for 8.0 in which binary logging is enabled by default already. Still, reduction may fail in STAGE9 as reducer has not been updated yet to handle this situation. As a workaround, add --log-bin to MYEXTRA, or simply stop at STAGE8 reduction, or add this functionality to STAGE9 and please push it back to the repository"

# To add this functionality, it is likely required to just handle the --server-id option removal using the BINLOG variable whilst setting --log-bin=0 at the same time or something - and this would be a good improvement for 8.0 (and beyond) testcase reduction in any case, as it would show/prove whetter it is necesary to have binlog on or not for a given testcase

fi

fi

fi

if [[ "${MYEXTRA}" == *"log"[-_]"bin"* ]]; then

if [[ ! "$(${BASEDIR}/bin/mysqld --version | grep -E --binary-files=text -oe '5\.[1567]|8\.[0-9]' | head -n1)" =~ ^5.[156]$ ]]; then # version is 5.7 or 8.0 and NOT 5.1, 5.5 or 5.6, i.e. --server-id is required

if [[ ! "${MYEXTRA}" == *"server"[-_]"id"* ]]; then

echo "Error: The version of mysqld is 5.7 or 8.0 and a --bin-log option was passed in MYEXTRA, yet no --server-id option was found whereas this is required for 5.7 and 8.0."

echo "Terminating now."

exit 1

fi

fi

BINLOG="$(echo "${MYEXTRA}" | grep -o "\-\-log[-_]bin[^ ]*" | head -n1)" # Grep all text including and after '--log[-_]bin' upto a space

MYEXTRA="$(echo "${MYEXTRA}" | sed "s|${BINLOG}||g")"

if [[ "${MYEXTRA}" == *"--server"[-_]"id"* ]]; then

BINLOGSI="$(echo "${MYEXTRA}" | grep -o "\-\-server[-_]id[^ ]*" | head -n1)" # Grep all text including and after '--server[-_]id' upto the first space

MYEXTRA="$(echo "${MYEXTRA}" | sed "s|${BINLOGSI}||g")"

BINLOG="$(echo "${BINLOG} ${BINLOGSI}")"

fi

if [[ "${MYEXTRA}" == *"server"[-_]"id"* ]]; then

echo "Error: --server-id seems to be present twice in MYEXTRA. Please remove at least one instance."

echo "Terminating now."

exit 1

fi

fi

# === Check for ONLY_FULL_GROUP_BY sql mode, split it into a ONLYFULLGROUPBY variable, and cleanup MYEXTRA to remove the option

ONLYFULLGROUPBY=

if [[ "${MYEXTRA}" != *"--sql_mode=ONLY_FULL_GROUP_BY,"* ]]; then # Avoid scenario where multiple sql_mode's are set, handling this would be mode complex (TODO)

if [[ "${MYEXTRA}" == *"--sql_mode=ONLY_FULL_GROUP_BY"* ]]; then

ONLYFULLGROUPBY="--sql_mode=ONLY_FULL_GROUP_BY"

MYEXTRA="$(echo "${MYEXTRA}" | sed "s|${ONLYFULLGROUPBY}||g")"

fi

fi

# ===== [SPECIAL MYEXTRA SECTION END]: Make sure to update 'SPECIAL_MYEXTRA_OPTIONS' re-declaration below if you add additional sections (i.e. MYEXTRA special option sets) above!

SPECIAL_MYEXTRA_OPTIONS="$TOKUDB $ROCKSDB $BL_ENCRYPTION $KF_ENCRYPTION $BINLOG $ONLYFULLGROUPBY"

SPECIAL_MYEXTRA_OPTIONS=$(echo $SPECIAL_MYEXTRA_OPTIONS | sed 's|^[ \t]\+||;s|[ \t]\+$||;s| | |g')

# For GLIBC crash reduction, we need to capture the output of the console from which reducer.sh is started. Currently only a SINGLE threaded solution using the 'scrip'

# binary from the util-linux package was found. The script binary is able to capture the GLIC output from the main console. It may be interesting to review the source C

# code for script, if available, to see how this is done. The following URL's may help with other possible solutions. However, LIBC_FATAL_STDERR_=1 has been found not to

# work at all. Perhaps this is due to mysqld being a setuid program (which is not confirmed). exec 2> and exec 1> redirections as well as process substitution i.e. >() have

# also been found not to work at all. Either the GLIBC trace will still be sent to the main console (and this may be the problem to start with), or it does not display at

# all when testing various interactions. To further complicate things, subshells $() and & may have interesting redirection dynamics. Many combinations were tried. Complex.

# * http://stackoverflow.com/questions/2821577/is-there-a-way-to-make-linux-cli-io-redirection-persistent

# * http://stackoverflow.com/questions/4616061/glibc-backtrace-cant-redirect-output-to-file

# * http://stackoverflow.com/questions/4290336/how-to-redirect-runtime-errors-to-stderr

# * https://sourceware.org/git/?p=glibc.git;a=patch;h=1327439fc6ef182c3ab8c69a55d3bee15b3c62a7

if [ $REDUCE_GLIBC_OR_SS_CRASHES -gt 0 ]; then

if [ ! -r $SCRIPT_LOC ]; then

echo "Error: REDUCE_GLIBC_OR_SS_CRASHES is activated, which requires the 'script' binary (part of the util-linux package), but this binary could not be found at $SCRIPT_LOC"

echo "Please install it and/or set the correct path/binary name using the SCRIPT_LOC variable."

echo "Terminating now."

exit 1

fi

# Ensure the output of this console is logged. For this, reducer.sh is restarted with self-logging activated using script

# With thanks, http://stackoverflow.com/a/26308092 from http://stackoverflow.com/questions/5985060/bash-script-using-script-command-from-a-bash-script-for-logging-a-session

if [ -z "$REDUCER_TYPESCRIPT" ]; then

TYPESCRIPT_UNIQUE_FILESUFFIX=$RANDOM$RANDOM

exec $SCRIPT_LOC -q -f /tmp/reducer_typescript${TYPESCRIPT_UNIQUE_FILESUFFIX}.log -c "REDUCER_TYPESCRIPT=1 TYPESCRIPT_UNIQUE_FILESUFFIX=${TYPESCRIPT_UNIQUE_FILESUFFIX} $0 $@"

fi

fi

# Sanitize input filenames which do not have a path specified by pointing to the current path. This ensures [Finish] output looks correct (ref $WORK_BUG_DIR)

if [[ "${INPUTFILE}" != *"/"* ]]; then INPUTFILE="./${INPUTFILE}"; fi;

echo_out(){

echo "$(date +'%F %T') $1"

if [ -r $WORKD/reducer.log ]; then echo "$(date +'%F %T') $1" >> $WORKD/reducer.log; fi

}

echo_out_overwrite(){

# Used for frequent on-screen updating when using threads etc.

echo -ne "$(date +'%F %T') $1\r"

}

ctrl_c(){

echo_out "[Abort] CTRL+C Was pressed. Dumping variable stack"

echo_out "[Abort] WORKD: $WORKD (reducer log @ $WORKD/reducer.log) | EPOCH: $EPOCH"

if [ -r $WORKO ]; then # If there were no issues found, $WORKO was never written

echo_out "[Abort] Best testcase thus far: $WORKO"

else

echo_out "[Abort] Best testcase thus far: $INPUTFILE (= input file; no optimizations were successful)"

fi

echo_out "[Abort] End of dump stack"

if [ $PXC_MOD -eq 1 ]; then

echo_out "[Abort] Ensuring any remaining PXC nodes are terminated and removed"

(ps -ef | grep -e 'node1_socket\|node2_socket\|node3_socket' | grep -v grep | grep $EPOCH | awk '{print $2}' | xargs kill -9 >/dev/null 2>&1 || true)

sleep 2; sync

fi

if [ $GRP_RPL_MOD -eq 1 ]; then

echo_out "[Abort] Ensuring any remaining Group Replication nodes are terminated and removed"

(ps -ef | grep -e 'node1_socket\|node2_socket\|node3_socket' | grep -v grep | grep $EPOCH | awk '{print $2}' | xargs kill -9 >/dev/null 2>&1 || true)

sleep 2; sync

fi

echo_out "[Abort] Ensuring any remaining processes are terminated"

if [ "$EPOCH" != "" ]; then

PIDS_TO_TERMINATE=$(ps -ef | grep -E --binary-files=text $WHOAMI | grep -E --binary-files=text $EPOCH | grep -E --binary-files=text -v "grep" | awk '{print $2}' | tr '\n' ' ')

else

echo_out "Assert: \$EPOCH is empty! in ctrl_c()!"

fi

echo_out "[Abort] Terminating these PID's: $PIDS_TO_TERMINATE"

kill -9 $PIDS_TO_TERMINATE >/dev/null 2>&1

echo_out "[Abort] What follows below is a call of finish(), the results are likely correct, but may be mangled due to the interruption"

finish

echo_out "[Abort] Done. Terminating reducer"

exit 2

}

options_check(){

# $1 to this procedure = $1 to the program - i.e. the SQL file to reduce

if [ "$1" == "" -a "$INPUTFILE" == "" ]; then

echo "Error: no input file given. Please give an SQL file to reduce as the first option to this script, or set inside the script as INPUTFILE=file_to_reduce.sql"

echo "Terminating now."

exit 1

fi

# Sudo check

if [ "$(sudo -A echo 'test' 2>/dev/null)" != "test" ]; then

echo "Error: sudo is not available or requires a password. This script needs to be able to use sudo, without password, from the userID that invokes it ($(whoami))"

echo "To get your setup correct, you may like to use a tool like visudo (use 'sudo visudo' or 'su' and then 'visudo') and consider adding the following line to the file:"

echo "$(whoami) ALL=(ALL) NOPASSWD:ALL"

echo "If you do not have sudo installed yet, try 'su' and then 'yum install sudo' or the apt-get equivalent"

echo "Terminating now."

exit 1

fi

# Note that instead of giving the SQL file on the cmd line, $INPUTFILE can be set (./process does so automaticaly using the #VARMOD# marker above)

if [ $(sysctl -n fs.aio-max-nr) -lt 300000 ]; then

echo "As fs.aio-max-nr on this system is lower than 300000, so you will likely run into BUG#12677594: INNODB: WARNING: IO_SETUP() FAILED WITH EAGAIN"

echo "To prevent this from happening, please use the following command at your shell prompt (you will need to have sudo privileges):"

echo "sudo sysctl -w fs.aio-max-nr=300000"

echo "The setting can be verified by executing: sysctl fs.aio-max-nr"

echo "Alternatively, you can add make the following settings to be system wide:"

echo "sudo vi /etc/sysctl.conf # Then, add the following two lines to the bottom of the file"

echo "fs.aio-max-nr = 1048576"

echo "fs.file-max = 6815744"

echo "Terminating now."

exit 1

fi

# Check if O_DIRECT is being used on tmpfs, which (when the original run was not on tmpfs) is not a 100% reproduce match, which may affect reproducibility

# See http://bugs.mysql.com/bug.php?id=26662 for more info

if $(echo $MYEXTRA | grep -E --binary-files=text -qi "MYEXTRA=.*O_DIRECT"); then

if [ $WORKDIR_LOCATION -eq 1 -o $WORKDIR_LOCATION -eq 2 ]; then # ramfs may not have this same issue, maybe '-o $WORKDIR_LOCATION -eq 2' can be removed?

echo 'Error: O_DIRECT is being used in the MYEXTRA option string, and tmpfs (or ramfs) storage was specified, but because'

echo 'of bug http://bugs.mysql.com/bug.php?id=26662 one would see a WARNING for this in the error log along the lines of;'

echo '[Warning] InnoDB: Failed to set O_DIRECT on file ./ibdata1: OPEN: Invalid argument, continuing anyway.'

echo " O_DIRECT is known to result in 'Invalid argument' on Linux on tmpfs, see MySQL Bug#26662."

echo 'So, reducer is exiting to allow you to change WORKDIR_LOCATION in the script to a non-tmpfs setting.'

echo 'Note: this assertion currently shows for ramfs as well, yet it has not been established if ramfs also' #

echo ' shows the same problem. If it does not (modify the script in this section to get it to run with ramfs' # ramfs, delete if ramfs is affected

echo ' as a trial/test), then please remove ramfs, or, if it does, then please remove these 3 last lines.' #

echo "Terminating now."

exit 1

fi

fi

# This section could be expanded to check for any directory specified (by for instance checking for paths), not just the two listed here

DIR_ISSUE=0

if $(echo $MYEXTRA | grep -E --binary-files=text -qi "MYEXTRA=.*innodb_log_group_home_dir"); then DIR_ISSUE='innodb_log_group_home_dir'; fi

if $(echo $MYEXTRA | grep -E --binary-files=text -qi "MYEXTRA=.*innodb_log_arch_dir"); then DIR_ISSUE='innodb_log_arch_dir'; fi

if [ "$DIR_ISSUE" != "0" ]; then

echo "Error: the $DIR_ISSUE option is being used in the MYEXTRA option string. This can lead to all sorts of problems;"

echo 'Remember that reducer 1) is multi-threaded - i.e. it would access that particularly named directory for each started mysqld, which'

echo 'clearly would result in issues, and 2) whilst reducer creates new directories for every trial (and for each thread), it would not do'

echo 'anything for this hardcoded directory, so this directory would get used every time, again clearly resulting in issues, especially'

echo 'when one considers that 3) running mysqld instances get killed once the achieved result (for example, issue discovered) is obtained.'

echo 'Suggested course of action: remove this/these sort of options from the MYEXTRA string and see if the issue reproduces. This/these sort'

echo 'of options often have little effect on reproducibility. Howerver, if found significant, reducer.sh can be expanded to cater for this/'

echo 'these sort of options being in MYEXTRA by re-directing them to a per-trial (and per-thread) subdirectory of the trial`s rundir used.'

echo 'Terminating reducer to allow this change to be made.'

exit 1

fi

if [ $MODE -ge 6 ]; then

if [ ! -d "$1" ]; then

echo 'Error: A file name was given as input, but a directory name was expected.'

echo "(MODE $MODE is set. Where you trying to use MODE 4 or lower?)"

echo "Terminating now."

exit 1

fi

if ! [ -d "$1/log/" -a -x "$1/log/" ]; then

echo 'Error: No input directory containing a "/log" subdirectory was given, or the input directory could not be read.'

echo 'Please specify a correct RQG vardir to reduce a multi-threaded testcase.'

echo 'Example: ./reducer /starfish/data_WL1/vardir1_1000 -> to reduce ThreadSync trial 1000'

echo "Terminating now."

exit 1

else

TS_THREADS=$(ls -l $1/log/C[0-9]*T[0-9]*.sql | wc -l | tr -d '[\t\n ]*')

# Making sure $TS_ELIMINATION_THREAD_ID is higher than number of threads to avoid 'unary operator expected' in cleanup_and_save during STAGE V

TS_ELIMINATION_THREAD_ID=$[$TS_THREADS+1]

if [ $TS_THREADS -lt 1 ]; then

echo 'Error: though input directory was found, no ThreadSync SQL trace files are present, or they could not be read.'

echo "Please check the directory at $1"

echo 'For the presence of 'C[0-9]*T[0-9]*.sql' files (for example, C1T10.sql).'

echo 'Note: a data load file (such as CT2.sql or CT3.sql) alone is not sufficient: thread sql data would be missing.'

echo "Terminating now."

exit 1

else

TS_INPUTDIR="$1/log"

TOKUDB_RUN_DETECTED=0

if echo "${SPECIAL_MYEXTRA_OPTIONS} ${MYEXTRA}" | grep -E --binary-files=text -qi "tokudb"; then TOKUDB_RUN_DETECTED=1; fi

if [ ${DISABLE_TOKUDB_AUTOLOAD} -eq 0 ]; then

if grep -E --binary-files=text -qi "tokudb" $TS_INPUTDIR/C[0-9]*T[0-9]*.sql; then TOKUDB_RUN_DETECTED=1; fi

fi

if [ ${TOKUDB_RUN_DETECTED} -eq 1 ]; then

if [ -r `sudo find /usr/*lib*/ -name libjemalloc.so.1 | head -n1` ]; then

export LD_PRELOAD=`sudo find /usr/*lib*/ -name libjemalloc.so.1 | head -n1`

else

if [ -r `sudo find /usr/local/*lib*/ -name libjemalloc.so.1 | head -n1` ]; then

export LD_PRELOAD=`sudo find /usr/local/*lib*/ -name libjemalloc.so.1 | head -n1`

else

echo 'This run contains TokuDB SE SQL, yet jemalloc - which is required for TokuDB - was not found, please install it first'

echo 'This can be done with a command similar to: $ yum install jemalloc'

echo "Terminating now."

exit 1

fi

fi

fi

fi

fi

else

if [ -d "$1" ]; then

echo 'Error: A directory was given as input, but a filename was expected.'

echo "(MODE $MODE is set. Where you trying to use MODE 6 or higher?)"

echo "Terminating now."

exit 1

fi

if [ ! -r "$1" ]; then

if [ ! -r $INPUTFILE ]; then

if [ "$INPUTFILE" == "" -a "$1" == "" ]; then

echo 'Error: No input file was given.'

else

echo 'Error: The specified input file did not exist or could not be read.'

fi

echo 'Please specify a single SQL file to reduce.'

echo 'Example: ./reducer ~/1.sql --> to process ~/1.sql'

echo 'Also, please ensure input file name only contains [0-9a-zA-Z_-] characters'

echo "For reference, this message was produced by $0"

echo 'Terminating now.'

exit 1

fi

else

export -n INPUTFILE=$1 # export -n is not necessary for this script, but it is here to prevent pquery-prep-red.sh from seeing this as a adjustable var

fi

TOKUDB_RUN_DETECTED=0

if echo "${SPECIAL_MYEXTRA_OPTIONS} ${MYEXTRA}" | grep -E --binary-files=text -qi "tokudb"; then TOKUDB_RUN_DETECTED=1; fi

if [ ${DISABLE_TOKUDB_AUTOLOAD} -eq 0 ]; then

if grep -E --binary-files=text -qi "tokudb" ${INPUTFILE}; then TOKUDB_RUN_DETECTED=1; fi

fi

if [ ${TOKUDB_RUN_DETECTED} -eq 1 ]; then

#if [ ${DISABLE_TOKUDB_AUTOLOAD} -eq 0 ]; then # Just here for extra safety

# if ! echo "${SPECIAL_MYEXTRA_OPTIONS} ${MYEXTRA}" | grep -E --binary-files=text -qi "plugin-load=tokudb=ha_tokudb.so"; then MYEXTRA="${MYEXTRA} --plugin-load=tokudb=ha_tokudb.so"; fi

# if ! echo "${SPECIAL_MYEXTRA_OPTIONS} ${MYEXTRA}" | grep -E --binary-files=text -qi "tokudb-check-jemalloc"; then MYEXTRA="${MYEXTRA} --tokudb-check-jemalloc=0"; fi

#fi

#if [ -r /usr/lib64/libjemalloc.so.1 ]; then

# export LD_PRELOAD=/usr/lib64/libjemalloc.so.1

if [ -r `sudo find /usr/*lib*/ -name libjemalloc.so.1 | head -n1` ]; then

export LD_PRELOAD=`sudo find /usr/*lib*/ -name libjemalloc.so.1 | head -n1`

else

if [ -r `sudo find /usr/local/*lib*/ -name libjemalloc.so.1 | head -n1` ]; then

export LD_PRELOAD=`sudo find /usr/local/*lib*/ -name libjemalloc.so.1 | head -n1`

else

echo 'This run contains TokuDB SE SQL, yet jemalloc - which is required for TokuDB - was not found, please install it first'

echo 'This can be done with a command similar to: $ yum install jemalloc'

echo "Terminating now."

exit 1

fi

fi

fi

fi

if [ $MODE -eq 0 ]; then

if [ "${TIMEOUT_COMMAND}" != "" ]; then

echo "Error: MODE is set to 0, and TIMEOUT_COMMAND is set. Both functions should not be used at the same time"

echo "Use either MODE=0 (and set TIMEOUT_CHECK), or TIMEOUT_COMMAND in combination with some other MODE, for example MODE=2 or MODE=3"

echo "Terminating now."

exit 1

fi

if [ ${TIMEOUT_CHECK} -le 30 ]; then

echo "Error: MODE=0 and TIMEOUT_CHECK<=30. When using MODE=0, set TIMEOUT_CHECK at least to: (2x the expected testcase duration lenght in seconds)+30 seconds extra!"

echo "Terminating now."

exit 1

fi

TIMEOUT_CHECK_REAL=$[ ${TIMEOUT_CHECK} - 30 ];

if [ ${TIMEOUT_CHECK_REAL} -le 0 ]; then

echo "Assert: TIMEOUT_CHECK_REAL<=0"

echo "Terminating now."

exit 1

fi

TIMEOUT_COMMAND="timeout --signal=SIGKILL ${TIMEOUT_CHECK}s" # TIMEOUT_COMMAND var is used (hack) instead of adding yet another MODE0 specific variable

fi

if [ "${TIMEOUT_COMMAND}" != "" -a "$(timeout 2>&1 | grep -E --binary-files=text -o 'information')" != "information" ]; then

echo "Error: TIMEOUT_COMMAND is set, yet the timeout command does not seem to be available"

echo "Terminating now."

exit 1

fi

if [ $MODE -eq 3 -a $USE_TEXT_STRING -eq 1 ]; then

if [ ! -r "$TEXT_STRING_LOC" ] ; then

echo "Assert: MODE=3 and USE_TEXT_STRING=1, so reducer.sh looked for $TEXT_STRING_LOC, but this file was either not found, or is not script readable"

exit

fi

fi

if [ $MODE -eq 2 ]; then

if [ $PQUERY_MOD -eq 1 ]; then # pquery client output testing run in MODE=2 - we need to make sure we have pquery client logging activated

if [ "$(echo $PQUERY_EXTRA_OPTIONS | grep -E --binary-files=text -io "log-client-output")" != "log-client-output" ]; then

echo "Assert: PQUERY_MOD=1 && PQUERY_EXTRA_OPTIONS does not contain log-client-output, so not sure what file reducer.sh should check for TEXT occurence."

exit 1

fi

fi

fi

BIN="${BASEDIR}/bin/mysqld"

if [ ! -s "${BIN}" ]; then

BIN="${BASEDIR}/bin/mysqld-debug"

if [ ! -s "${BIN}" ]; then

echo "Assert: No mysqld or mysqld-debug binary was found in ${BASEDIR}/bin"

echo 'Please check script contents/options and set the $BASEDIR variable correctly'

echo "Terminating now."

exit 1

fi

fi

if [ $MODE -ne 0 -a $MODE -ne 1 -a $MODE -ne 2 -a $MODE -ne 3 -a $MODE -ne 4 -a $MODE -ne 5 -a $MODE -ne 6 -a $MODE -ne 7 -a $MODE -ne 8 -a $MODE -ne 9 ]; then

echo "Error: Invalid MODE set: $MODE (valid range: 1-9)"

echo 'Please check script contents/options ($MODE variable)'

echo "Terminating now."

exit 1

fi

if [ $MODE -eq 1 -o $MODE -eq 2 -o $MODE -eq 3 -o $MODE -eq 5 -o $MODE -eq 6 -o $MODE -eq 7 -o $MODE -eq 8 ]; then

if [ ! -n "$TEXT" ]; then

echo "Error: MODE set to $MODE, but no \$TEXT variable was defined, or \$TEXT is blank"

echo 'Please check script contents/options ($TEXT variable)'

echo "Terminating now."

exit 1

fi

fi

if [[ $PXC_MOD -eq 1 || $GRP_RPL_MOD -eq 1 ]]; then

PQUERY_MOD=1

# ========= These are currently limitations of PXC/Group Replication mode. Feel free to extend reducer.sh to handle these ========

#export -n MYEXTRA="" # Serious shortcoming. Work to be done. PQUERY MYEXTRA variables will be added docker-compose.yml

export -n FORCE_SPORADIC=0

export -n SPORADIC=0

export -n FORCE_SKIPV=0

export -n SKIPV=1

export -n MULTI_THREADS=0 # Original thought here was to avoid dozens of 3-container docker setups. This needs reviewing now that mysqld is used directly.

# /==========

if [ $MODE -eq 0 ]; then

echo "Error: PXC/Group Replication mode is set to 1, and MODE=0 set to 0, but this option combination has not been tested/added to reducer.sh yet. Please do so!"

echo "Terminating now."

exit 1

fi

if [ "${TIMEOUT_COMMAND}" != "" ]; then

echo "Error: PXC/Group Replication mode is set to 1, and TIMEOUT_COMMAND is set, but this option combination has not been tested/added to reducer.sh yet. Please do so!"

echo "Terminating now."

exit 1

fi

if [ $MODE -eq 1 -o $MODE -eq 6 ]; then

echo "Error: Valgrind for 3 node PXC/Group Replication replay has not been implemented yet. Please do so! Free cookies afterwards!"

echo "Terminating now."

exit 1

fi

if [ $MODE -ge 6 -a $MODE -le 9 ]; then

echo "Error: wrong option combination: MODE is set to $MODE (ThreadSync) and PXC/Group Replication mode is active"

echo 'Please check script contents/options ($MODE and $PXC mode variables)'

echo "Terminating now."

exit 1

fi

fi

if [ $PQUERY_MULTI -eq 1 ]; then

PQUERY_MOD=1

fi

if [ $PQUERY_MOD -eq 1 ]; then

if [ ! -r "$PQUERY_LOC" ]; then

echo "Error: PQUERY_MOD is set to 1, but the pquery binary (as defined by PQUERY_LOC; currently set to '$PQUERY_LOC') is not available."

echo 'Please check script contents/options ($PQUERY_MOD and $PQUERY_LOC variables)'

echo "Terminating now."

exit 1

fi

fi

if [ $PQUERY_MULTI -gt 0 ]; then

export -n FORCE_SKIPV=1

MULTI_THREADS=$PQUERY_MULTI_THREADS

if [ $PQUERY_MULTI_CLIENT_THREADS -lt 1 ]; then

echo_out "Error: PQUERY_MULTI_CLIENT_THREADS is set to less then 1 ($PQUERY_MULTI_CLIENT_THREADS), while PQUERY_MULTI active, this does not work; reducer needs threads to be able to replay the issue"

echo "Terminating now."

exit 1

elif [ $PQUERY_MULTI_CLIENT_THREADS -eq 1 ]; then

echo_out "Warning: PQUERY_MULTI active, and PQUERY_MULTI_CLIENT_THREADS is set to 1; 1 thread for a multi-threaded issue does not seem logical. Proceeding, but this is highly likely incorrect. Please check. NOTE: There is at least one possible use case for this: proving that a sporadic mysqld startup can be reproduced (with a near-empty SQL file; i.e. the run is concerned with reproducing the startup issue, not reducing the SQL file)"

elif [ $PQUERY_MULTI_CLIENT_THREADS -lt 5 ]; then

echo_out "Warning: PQUERY_MULTI active, and PQUERY_MULTI_CLIENT_THREADS is set to $PQUERY_MULTI_CLIENT_THREADS, $PQUERY_MULTI_CLIENT_THREADS threads for reproducing a multi-threaded issue via random replay seems insufficient. You may want to increase PQUERY_MULTI_CLIENT_THREADS. Proceeding, but this is likely incorrect. Please check"

fi

fi

if [ $REDUCE_GLIBC_OR_SS_CRASHES -gt 0 ]; then

export -n MULTI_THREADS=1 # Likely not needed, because MULTI mode should never become active for REDUCE_GLIBC_OR_SS_CRASHES=1 (and there is a matching assert),

export -n MULTI_THREADS_INCREASE=0 # so it is here as a safety measure only FTM.

export -n SLOW_DOWN_CHUNK_SCALING=1

export -n SKIPV=1

if [ $MODE -ne 3 -a $MODE -ne 4 ]; then

echo "REDUCE_GLIBC_OR_SS_CRASHES is active, and MODE is set to MODE=$MODE, which is not supported (yet). Currently only modes 3 and 4 are supported when reducing GLIBC crashes"

echo "Terminating now."

exit 1

fi

if [[ $PXC_MOD -gt 0 || $GRP_RPL_MOD -eq 1 ]]; then

echo "GLIBC testcase reduction is not yet supported for PXC_MOD=1 or GRP_RPL_MOD=1. This would be very complex to code, except perhaps for a single node cluster or for one node only. See source code for details. Search for 'GLIBC crash reduction'"

echo "A workaround may be to see if this GLIBC crash reproduces on standard (non-cluster) mysqld also, which is likely."

echo "Terminating now."

exit 1

fi

if [ $PQUERY_MULTI -gt 0 ]; then

echo "GLIBC testcase reduction is not yet supported for PQUERY_MULTI=1. This would be very complex to code. See source code for details. Search for 'GLIBC crash reduction'"

echo "A workaround may be to see if this GLIBC crash reproduces using a single threaded execution, which in most cases is somewhat likely."

echo "Terminating now."

exit 1

fi

fi

if [ $FORCE_SKIPV -gt 0 ]; then

export -n FORCE_SPORADIC=1

export -n SKIPV=1

fi

if [ $FORCE_SPORADIC -gt 0 ]; then

if [ $STAGE1_LINES -eq 90 ]; then # Do not change any customized/non-default (i.e. !=90) setting as this may be handy for automation. For example, pquery-reach.sh will set STAGE1_LINES to 13 while activating FORCE_SKIPV=1 which means that reducer will reduce in MULTI (multi-threaded subreducer) mode until 13 lines are reached, then it will swap to single threaded. This is great to manage a combination of both sporadic (they will be reduced to at max 13 lines) and static (they will be full reduced) issues.

export -n STAGE1_LINES=3

fi

export -n SPORADIC=1

export -n SLOW_DOWN_CHUNK_SCALING=1

fi

if [ $MODE -eq 0 -a $FORCE_KILL=1 ]; then

FORCE_KILL=0

fi

export -n MYEXTRA=`echo ${MYEXTRA} | sed 's|[ \t]*--no-defaults[ \t]*||g'` # Ensuring --no-defaults is no longer part of MYEXTRA. Reducer already sets this itself always.

}

remove_dropc(){

if [ "$1" == "" ]; then

echo_out "Assert: no parameter was passed to the remove_dropc() function. This should not happen."

exit 1

fi

# Loop through the top of the passed file (usually WORKT or WORKF) and remove all seen individual DROPC lines

# Implenting things this way became necessary once individual lines were used for DROPC instead of just one long line

# which could be grepped out with grep -v. The reason for having to use individual lines for DROPC is that pquery

# will not process STATEMENT1;STATEMENT2; and this led to errors. This is only done for PQUERY_MOD=1 runs to ensure

# backwards compatibility (i.e. the mysql client will still use grep -v DROPC instead)

while :; do

DROPC_LINE_REMOVED=0

if [[ "$(cat $1 | head -n1)" == *"DROP DATABASE transforms;"* ]]; then

sed -i '1d' $1

DROPC_LINE_REMOVED=1

fi

if [[ "$(cat $1 | head -n1)" == *"CREATE DATABASE transforms;"* ]]; then

sed -i '1d' $1

DROPC_LINE_REMOVED=1

fi

if [[ "$(cat $1 | head -n1)" == *"DROP DATABASE test;"* ]]; then

sed -i '1d' $1

DROPC_LINE_REMOVED=1

fi

if [[ "$(cat $1 | head -n1)" == *"CREATE DATABASE test;"* ]]; then

sed -i '1d' $1

DROPC_LINE_REMOVED=1

fi

if [[ "$(cat $1 | head -n1)" == *"USE test;"* ]]; then

sed -i '1d' $1

DROPC_LINE_REMOVED=1

fi

if [ $DROPC_LINE_REMOVED -eq 0 ]; then

break

fi

done

}

set_internal_options(){ # Internal options: do not modify!

# Try and raise max user processes limit (please also preset the soft/hard nproc settings in /etc/security/limits.conf (Centos), both to at least 20480 - see percona-qa/setup_server.sh for an example)

ulimit -u 4000 2>/dev/null # Attempt to raise it to 4000

ulimit -u 10000 2>/dev/null # Attempt to raise it even higher, if it fails, but the previous one worked, then that one is still used

ulimit -u 20000 2>/dev/null # Attempt to raise it even higher, if it fails, but a previous one worked, then that one is still used

ulimit -u 30000 2>/dev/null # Attempt to raise it even higher, if it fails, but a previous one worked, then that one is still used

# Unless core files are specifically requested (--core-file or --core option passed to mysqld via MYEXTRA), disable all core file generation (OS+mysqld)

# It would be good if we could disable OS core file generation without disabling mysqld core file generation, but for the moment it looks like

# ulimit -c 0 disables ALL core file generation, both OS and mysqld, so instead, ftm, reducer checks for "CORE" in MYEXTRA (uppercase-ed via ^^)

# and if present reducer does not disable core file generation (OS nor mysqld)

if [[ "${MYEXTRA^^}" != *"CORE"* ]]; then # ^^ = Uppercase MYEXTRA contents before compare

ulimit -c 0 >/dev/null