Bootstrap for develop-6

This commit is contained in:
Leonid Fedorov
2022-06-07 09:20:46 +00:00
parent 5682a1f245
commit 92f759f2ee
2 changed files with 682 additions and 0 deletions

305
build/bootstrap_mcs.sh Executable file
View File

@ -0,0 +1,305 @@
#!/bin/bash
# This script compiles/installs MCS from scratch and it makes some assumptions:
# - the server's source code is two directories above the MCS engine source.
# - the script is to be run under root.
SCRIPT_LOCATION=$(dirname "$0")
MDB_SOURCE_PATH=$(realpath $SCRIPT_LOCATION/../../../..)
source $SCRIPT_LOCATION/utils.sh
if [ "$EUID" -ne 0 ]
then error "Please run script as root to install MariaDb to system paths"
exit 1
fi
message "Building Mariadb Server from $color_yellow$MDB_SOURCE_PATH$color_normal"
BUILD_TYPE_OPTIONS=("Debug" "RelWithDebInfo")
DISTRO_OPTIONS=("Ubuntu" "CentOS" "Debian" "openSUSE" "Rocky")
BRANCHES=($(git branch --list --no-color| grep "[^* ]+" -Eo))
optparse.define short=t long=build-type desc="Build Type: ${BUILD_TYPE_OPTIONS[*]}" variable=MCS_BUILD_TYPE
optparse.define short=d long=distro desc="Choouse your OS: ${DISTRO_OPTIONS[*]}" variable=OS
optparse.define short=s long=skip-deps desc="Skip install dependences" variable=SKIP_DEPS default=false value=true
optparse.define short=C long=force-cmake-reconfig desc="Force cmake reconfigure" variable=FORCE_CMAKE_CONFIG default=false value=true
optparse.define short=S long=skip-columnstore-submodules desc="Skip columnstore submodules initialization" variable=SKIP_SUBMODULES default=false value=true
optparse.define short=b long=branch desc="Choouse git branch ('none' for menu)" variable=BRANCH
source $( optparse.build )
if [[ ! " ${BUILD_TYPE_OPTIONS[*]} " =~ " ${MCS_BUILD_TYPE} " ]]; then
getChoice -q "Select your Build Type" -o BUILD_TYPE_OPTIONS
MCS_BUILD_TYPE=$selectedChoice
fi
if [[ ! " ${DISTRO_OPTIONS[*]} " =~ " ${OS} " || $OS = "CentOS" ]]; then
detect_distro
fi
INSTALL_PREFIX="/usr/"
DATA_DIR="/var/lib/mysql/data"
CMAKE_BIN_NAME=cmake
select_branch()
{
if [[ ! " ${BRANCHES[*]} " =~ " ${BRANCH} " ]]; then
if [[ $BRANCH = 'none' ]]; then
getChoice -q "Select your branch" -o BRANCHES
BRANCH=$selectedChoice
fi
cd $SCRIPT_LOCATION
message "Selecting $BRANCH branch for Columnstore"
git checkout $BRANCH
cd -
message "Turning off Columnstore submodule auto update via gitconfig"
cd $MDB_SOURCE_PATH
git config submodule.storage/columnstore/columnstore.update none
cd -
fi
cd $SCRIPT_LOCATION
CURRENT_BRANCH=$(git branch --show-current)
cd -
message "Columnstore will be built from $color_yellow$CURRENT_BRANCH$color_normal branch"
}
install_deps()
{
message "Installing deps"
if [[ $OS = 'Ubuntu' || $OS = 'Debian' ]]; then
sudo apt-get -y update
apt-get -y install build-essential automake libboost-all-dev bison cmake \
libncurses5-dev libaio-dev libsystemd-dev libpcre2-dev \
libperl-dev libssl-dev libxml2-dev libkrb5-dev flex libpam-dev git \
libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest libsnappy-dev libjemalloc-dev
elif [[ $OS = 'CentOS' || $OS = 'Rocky' ]]; then
yum -y install epel-release \
&& yum -y groupinstall "Development Tools" \
&& yum config-manager --set-enabled powertools \
&& yum -y install bison ncurses-devel readline-devel perl-devel openssl-devel libxml2-devel gperf libaio-devel libevent-devel tree wget pam-devel snappy-devel libicu \
&& yum -y install vim wget strace ltrace gdb rsyslog net-tools openssh-server expect boost perl-DBI libicu boost-devel initscripts jemalloc-devel libcurl-devel gtest-devel cppunit-devel systemd-devel
if [[ "$OS_VERSION" == "7" ]]; then
yum -y install cmake3
CMAKE_BIN_NAME=cmake3
else
yum -y install cmake
fi
if [ $OS = 'Rocky' ]; then
yum install -y checkpolicy
fi
elif [ $OS = 'openSUSE' ]; then
zypper install -y bison ncurses-devel readline-devel libopenssl-devel cmake libxml2-devel gperf libaio-devel libevent-devel python-devel ruby-devel tree wget pam-devel snappy-devel libicu-devel \
&& zypper install -y libboost_system-devel libboost_filesystem-devel libboost_thread-devel libboost_regex-devel libboost_date_time-devel libboost_chrono-devel libboost_atomic-devel \
&& zypper install -y vim wget strace ltrace gdb rsyslog net-tools expect perl-DBI libicu boost-devel jemalloc-devel libcurl-devel liblz4-devel lz4 \
&& zypper install -y gcc gcc-c++ git automake libtool gtest cppunit-devel pcre2-devel systemd-devel libaio-devel snappy-devel cracklib-devel policycoreutils-devel ncurses-devel \
&& zypper install -y make flex libcurl-devel automake libtool rpm-build lsof iproute pam-devel perl-DBI expect createrepo
fi
}
stop_service()
{
message "Stopping MariaDB services"
systemctl stop mariadb
systemctl stop mariadb-columnstore
}
check_service()
{
if systemctl is-active --quiet $1; then
message "$1 service started$color_green OK $color_normal"
else
error "$1 service failed"
service $1 status
fi
}
start_service()
{
message "Starting MariaDB services"
systemctl start mariadb-columnstore
systemctl start mariadb
check_service mariadb-columnstore
check_service mariadb
}
clean_old_installation()
{
message "Cleaning old installation"
rm -rf /var/lib/columnstore/data1/*
rm -rf /var/lib/columnstore/data/
rm -rf /var/lib/columnstore/local/
rm -f /var/lib/columnstore/storagemanager/storagemanager-lock
rm -f /var/lib/columnstore/storagemanager/cs-initialized
rm -rf /tmp/*
rm -rf /var/lib/mysql
rm -rf /var/run/mysqld
rm -rf $DATA_DIR
rm -rf /etc/mysql
rm -r /etc/my.cnf.d/provider_*
}
build()
{
message "Building sources in $color_yellow$MCS_BUILD_TYPE$color_normal mode"
local MDB_CMAKE_FLAGS="-DWITH_SYSTEMD=yes
-DPLUGIN_COLUMNSTORE=YES
-DPLUGIN_MROONGA=NO
-DPLUGIN_ROCKSDB=NO
-DPLUGIN_TOKUDB=NO
-DPLUGIN_CONNECT=NO
-DPLUGIN_SPIDER=NO
-DPLUGIN_OQGRAPH=NO
-DPLUGIN_SPHINX=NO
-DWITH_EMBEDDED_SERVER=OFF
-DBUILD_CONFIG=mysql_release
-DWITH_WSREP=OFF
-DWITH_SSL=system
-DWITH_UNITTESTS=YES
-DWITH_BRM_UT=YES
-DCMAKE_INSTALL_PREFIX:PATH=$INSTALL_PREFIX"
cd $MDB_SOURCE_PATH
if [[ $SKIP_SUBMODULES = true ]] ; then
warn "Skipping initialization of columnstore submodules"
else
message "Initialization of columnstore submodules"
cd storage/columnstore/columnstore
git submodule update --init
cd -
fi
if [[ $FORCE_CMAKE_CONFIG = true ]] ; then
warn "Erasing cmake cache"
rm -f "$MDB_SOURCE_PATH/CMakeCache.txt"
rm -rf "$MDB_SOURCE_PATH/CMakeFiles"
fi
if [[ "$OS" = 'Ubuntu' || "$OS" = 'Debian' ]]; then
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DDEB=bionic"
elif [ $OS = 'CentOS' ]; then
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DRPM=CentOS7"
elif [ $OS = 'Rocky' ]; then
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DRPM=CentOS7"
elif [ $OS = 'openSUSE' ]; then
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DRPM=sles15"
fi
message "building with flags $MDB_CMAKE_FLAGS"
local CPUS=$(getconf _NPROCESSORS_ONLN)
${CMAKE_BIN_NAME} . -DCMAKE_BUILD_TYPE=$MCS_BUILD_TYPE $MDB_CMAKE_FLAGS && \
make -j $CPUS install
if [ $? -ne 0 ]; then
error "!!!! BUILD FAILED"
exit 1
fi
cd -
}
check_user_and_group()
{
if [ -z "$(grep mysql /etc/passwd)" ]; then
message "Adding user mysql into /etc/passwd"
useradd -r -U mysql -d /var/lib/mysql
fi
if [ -z "$(grep mysql /etc/group)" ]; then
GroupID = `awk -F: '{uid[$3]=1}END{for(x=100; x<=999; x++) {if(uid[x] != ""){}else{print x; exit;}}}' /etc/group`
message "Adding group mysql with id $GroupID"
groupadd -g GroupID mysql
fi
}
install()
{
message "Installing MariaDB"
check_user_and_group
mkdir -p /etc/my.cnf.d
bash -c 'echo "[client-server]
socket=/run/mysqld/mysqld.sock" > /etc/my.cnf.d/socket.cnf'
mv $INSTALL_PREFIX/lib/mysql/plugin/ha_columnstore.so /tmp/ha_columnstore_1.so || mv $INSTALL_PREFIX/lib64/mysql/plugin/ha_columnstore.so /tmp/ha_columnstore_2.so
message "Running mysql_install_db"
mysql_install_db --rpm --user=mysql
mv /tmp/ha_columnstore_1.so $INSTALL_PREFIX/lib/mysql/plugin/ha_columnstore.so || mv /tmp/ha_columnstore_2.so $INSTALL_PREFIX/lib64/mysql/plugin/ha_columnstore.so
mkdir -p /etc/columnstore
cp $MDB_SOURCE_PATH/storage/columnstore/columnstore/oam/etc/Columnstore.xml /etc/columnstore/Columnstore.xml
cp $MDB_SOURCE_PATH/storage/columnstore/columnstore/storage-manager/storagemanager.cnf /etc/columnstore/storagemanager.cnf
cp $MDB_SOURCE_PATH/support-files/*.service /lib/systemd/system/
cp $MDB_SOURCE_PATH/storage/columnstore/columnstore/oam/install_scripts/*.service /lib/systemd/system/
if [[ "$OS" = 'Ubuntu' || "$OS" = 'Debian' ]]; then
mkdir -p /usr/share/mysql
mkdir -p /etc/mysql/
cp $MDB_SOURCE_PATH/debian/additions/debian-start.inc.sh /usr/share/mysql/debian-start.inc.sh
cp $MDB_SOURCE_PATH/debian/additions/debian-start /etc/mysql/debian-start
> /etc/mysql/debian.cnf
fi
systemctl daemon-reload
if [ -d "/etc/mysql/mariadb.conf.d/" ]; then
message "Copying configs from /etc/mysql/mariadb.conf.d/ to /etc/my.cnf.d"
cp -rp /etc/mysql/mariadb.conf.d/* /etc/my.cnf.d
fi
if [ -d "/etc/mysql/conf.d/" ]; then
message "Copying configs from /etc/mysql/conf.d/ to /etc/my.cnf.d"
cp -rp /etc/mysql/conf.d/* /etc/my.cnf.d
fi
mkdir -p /var/lib/columnstore/data1
mkdir -p /var/lib/columnstore/data1/systemFiles
mkdir -p /var/lib/columnstore/data1/systemFiles/dbrm
mkdir -p /run/mysqld/
mkdir -p $DATA_DIR
chown -R mysql:mysql $DATA_DIR
chown -R mysql:mysql /var/lib/columnstore/
chown -R mysql:mysql /run/mysqld/
chmod +x $INSTALL_PREFIX/bin/mariadb*
ldconfig
message "Running columnstore-post-install"
mkdir -p /var/lib/columnstore/local
columnstore-post-install --rpmmode=install
message "Running install_mcs_mysql"
install_mcs_mysql.sh
chown -R syslog:syslog /var/log/mariadb/
chmod 777 /var/log/mariadb/
chmod 777 /var/log/mariadb/columnstore
}
select_branch
if [[ $SKIP_DEPS = false ]] ; then
install_deps
fi
stop_service
clean_old_installation
build
install
start_service
message "$color_green FINISHED $color_normal"

377
build/utils.sh Normal file
View File

@ -0,0 +1,377 @@
color_normal=$(tput sgr0)
color_bold=$(tput bold)
color_red="$color_bold$(tput setaf 1)"
color_green=$(tput setaf 2)
color_fawn=$(tput setaf 3); color_beige="$color_fawn"
color_yellow="$color_bold$color_fawn"
color_darkblue=$(tput setaf 4)
color_blue="$color_bold$color_darkblue"
color_purple=$(tput setaf 5); color_magenta="$color_purple"
color_pink="$color_bold$color_purple"
color_darkcyan=$(tput setaf 6)
color_cyan="$color_bold$color_darkcyan"
color_gray=$(tput setaf 7)
color_darkgray="$color_bold"$(tput setaf 0)
color_white="$color_bold$color_gray"
message()
{
echo $color_cyan -- $@$color_normal
}
warn()
{
echo $color_yellow -- $@$color_normal
}
error()
{
echo $color_red -- $@$color_normal
}
detect_distro()
{
if [ -f /etc/os-release ]; then
. /etc/os-release
export OS=$NAME
export OS_VERSION=$VERSION_ID
elif type lsb_release >/dev/null 2>&1; then
# linuxbase.org
export OS=$(lsb_release -si)
export OS_VERSION=$(lsb_release -sr)
elif [ -f /etc/lsb-release ]; then
# For some versions of Debian/Ubuntu without lsb_release command
. /etc/lsb-release
export OS=$DISTRIB_ID
OS_VERSION=$DISTRIB_RELEASE
elif [ -f /etc/debian_version ]; then
# Older Debian/Ubuntu/etc.
OS=Debian
OS_VERSION=$(cat /etc/debian_version)
else
# Fall back to uname, e.g. "Linux <version>", also works for BSD, etc.
OS=$(uname -s)
OS_VERSION=$(uname -r)
fi
OS=$(echo $OS | cut -f 1 -d " ")
message "Detected $color_yellow$OS $OS_VERSION$color_normal"
}
menuStr=""
function hideCursor(){
printf "\033[?25l"
# capture CTRL+C so cursor can be reset
trap "showCursor && exit 0" 2
}
function showCursor(){
printf "\033[?25h"
}
function clearLastMenu(){
local msgLineCount=$(printf "$menuStr" | wc -l)
# moves the curser up N lines so the output overwrites it
echo -en "\033[${msgLineCount}A"
# clear to end of screen to ensure there's no text left behind from previous input
[ $1 ] && tput ed
}
function renderMenu(){
local start=0
local selector=""
local instruction="$1"
local selectedIndex=$2
local listLength=$itemsLength
local longest=0
local spaces=""
menuStr="\n $instruction\n"
# Get the longest item from the list so that we know how many spaces to add
# to ensure there's no overlap from longer items when a list is scrolling up or down.
for (( i=0; i<$itemsLength; i++ )); do
if (( ${#menuItems[i]} > longest )); then
longest=${#menuItems[i]}
fi
done
spaces=$(printf ' %.0s' $(eval "echo {1.."$(($longest))"}"))
if [ $3 -ne 0 ]; then
listLength=$3
if [ $selectedIndex -ge $listLength ]; then
start=$(($selectedIndex+1-$listLength))
listLength=$(($selectedIndex+1))
fi
fi
for (( i=$start; i<$listLength; i++ )); do
local currItem="${menuItems[i]}"
currItemLength=${#currItem}
if [[ $i = $selectedIndex ]]; then
selectedChoice="${currItem}"
selector="${color_green}${color_normal}"
currItem="${color_green}${currItem}${color_normal}"
else
selector=" "
fi
currItem="${spaces:0:0}${currItem}${spaces:currItemLength}"
menuStr="${menuStr}\n ${selector} ${currItem}"
done
menuStr="${menuStr}\n"
# whether or not to overwrite the previous menu output
[ $4 ] && clearLastMenu
printf "${menuStr}"
}
function getChoice(){
local KEY__ARROW_UP=$(echo -e "\033[A")
local KEY__ARROW_DOWN=$(echo -e "\033[B")
local KEY__ENTER=$(echo -e "\n")
local captureInput=true
local displayHelp=false
local maxViewable=0
local instruction="Select an item from the list:"
local selectedIndex=0
remainingArgs=()
while [[ $# -gt 0 ]]; do
key="$1"
case $key in
-h|--help)
displayHelp=true
shift
;;
-i|--index)
selectedIndex=$2
shift 2
;;
-m|--max)
maxViewable=$2
shift 2
;;
-o|--options)
menuItems=$2[@]
menuItems=("${!menuItems}")
shift 2
;;
-q|--query)
instruction="$2"
shift 2
;;
*)
remainingArgs+=("$1")
shift
;;
esac
done
# just display help
if $displayHelp; then
echo;
echo "Usage: getChoice [OPTION]..."
echo "Renders a keyboard navigable menu with a visual indicator of what's selected."
echo;
echo " -h, --help Displays this message"
echo " -i, --index The initially selected index for the options"
echo " -m, --max Limit how many options are displayed"
echo " -o, --options An Array of options for a User to choose from"
echo " -q, --query Question or statement presented to the User"
echo;
echo "Example:"
echo " foodOptions=(\"pizza\" \"burgers\" \"chinese\" \"sushi\" \"thai\" \"italian\" \"shit\")"
echo;
echo " getChoice -q \"What do you feel like eating?\" -o foodOptions -i \$((\${#foodOptions[@]}-1)) -m 4"
echo " printf \"\\n First choice is '\${selectedChoice}'\\n\""
echo;
echo " getChoice -q \"Select another option in case the first isn't available\" -o foodOptions"
echo " printf \"\\n Second choice is '\${selectedChoice}'\\n\""
echo;
return 0
fi
set -- "${remainingArgs[@]}"
local itemsLength=${#menuItems[@]}
# no menu items, at least 1 required
if [[ $itemsLength -lt 1 ]]; then
printf "\n [ERROR] No menu items provided\n"
exit 1
fi
renderMenu "$instruction" $selectedIndex $maxViewable
hideCursor
while $captureInput; do
read -rsn3 key # `3` captures the escape (\033'), bracket ([), & type (A) characters.
case "$key" in
"$KEY__ARROW_UP")
selectedIndex=$((selectedIndex-1))
(( $selectedIndex < 0 )) && selectedIndex=$((itemsLength-1))
renderMenu "$instruction" $selectedIndex $maxViewable true
;;
"$KEY__ARROW_DOWN")
selectedIndex=$((selectedIndex+1))
(( $selectedIndex == $itemsLength )) && selectedIndex=0
renderMenu "$instruction" $selectedIndex $maxViewable true
;;
"$KEY__ENTER")
clearLastMenu true
showCursor
captureInput=false
;;
esac
done
}
function optparse.throw_error(){
local message="$1"
error "OPTPARSE: ERROR: $message"
exit 1
}
# -----------------------------------------------------------------------------------------------------------------------------
function optparse.define(){
if [ $# -lt 3 ]; then
optparse.throw_error "optparse.define <short> <long> <variable> [<desc>] [<default>] [<value>]"
fi
for option_id in $( seq 1 $# ) ; do
local option="$( eval "echo \$$option_id")"
local key="$( echo $option | awk -F "=" '{print $1}' )";
local value="$( echo $option | awk -F "=" '{print $2}' )";
#essentials: shortname, longname, description
if [ "$key" = "short" ]; then
local shortname="$value"
if [ ${#shortname} -ne 1 ]; then
optparse.throw_error "short name expected to be one character long"
fi
local short="-${shortname}"
elif [ "$key" = "long" ]; then
local longname="$value"
if [ ${#longname} -lt 2 ]; then
optparse.throw_error "long name expected to be atleast one character long"
fi
local long="--${longname}"
elif [ "$key" = "desc" ]; then
local desc="$value"
elif [ "$key" = "default" ]; then
local default="$value"
elif [ "$key" = "variable" ]; then
local variable="$value"
elif [ "$key" = "value" ]; then
local val="$value"
fi
done
if [ "$variable" = "" ]; then
optparse.throw_error "You must give a variable for option: ($short/$long)"
fi
if [ "$val" = "" ]; then
val="\$OPTARG"
fi
# build OPTIONS and help
optparse_usage="${optparse_usage}#NL#TB${short} $(printf "%-25s %s" "${long}:" "${desc}")"
if [ "$default" != "" ]; then
optparse_usage="${optparse_usage} [default:$default]"
fi
optparse_contractions="${optparse_contractions}#NL#TB#TB${long})#NL#TB#TB#TBparams=\"\$params ${short}\";;"
if [ "$default" != "" ]; then
optparse_defaults="${optparse_defaults}#NL${variable}=${default}"
fi
optparse_arguments_string="${optparse_arguments_string}${shortname}"
if [ "$val" = "\$OPTARG" ]; then
optparse_arguments_string="${optparse_arguments_string}:"
fi
optparse_process="${optparse_process}#NL#TB#TB${shortname})#NL#TB#TB#TB${variable}=\"$val\";;"
}
# -----------------------------------------------------------------------------------------------------------------------------
function optparse.build(){
local build_file="$(mktemp -t "optparse-XXXXXX.tmp")"
# Building getopts header here
# Function usage
cat << EOF > $build_file
function usage(){
cat << XXX
usage: \$0 [OPTIONS]
OPTIONS:
$optparse_usage
-? --help : usage
XXX
}
# Contract long options into short options
params=""
while [ \$# -ne 0 ]; do
param="\$1"
shift
case "\$param" in
$optparse_contractions
"-?"|--help)
usage
exit 0;;
*)
if [[ "\$param" == --* ]]; then
echo -e "Unrecognized long option: \$param"
usage
exit 1
fi
params="\$params \"\$param\"";;
esac
done
eval set -- "\$params"
# Set default variable values
$optparse_defaults
# Process using getopts
while getopts "$optparse_arguments_string" option; do
case \$option in
# Substitute actions for different variables
$optparse_process
:)
echo "Option - \$OPTARG requires an argument"
exit 1;;
*)
usage
exit 1;;
esac
done
# Clean up after self
rm $build_file
EOF
local -A o=( ['#NL']='\n' ['#TB']='\t' )
for i in "${!o[@]}"; do
sed -i "s/${i}/${o[$i]}/g" $build_file
done
# Unset global variables
unset optparse_usage
unset optparse_process
unset optparse_arguments_string
unset optparse_defaults
unset optparse_contractions
# Return file name to parent
echo "$build_file"
}