grenade.sh

<stdin>

grenade.sh is an OpenStack upgrade test harness to exercise the OpenStack upgrade process. It uses DevStack to perform the initial OpenStack install.

# DIVIDER #!/usr/bin/env bash # DIVIDER # DIVIDER # DIVIDER export GRENADE_DIR=$(cd $(dirname "$0") && pwd) # DIVIDER export DSTOOLS_VERSION=${DSTOOLS_VERSION:-0.4.0} source $GRENADE_DIR/grenaderc source $GRENADE_DIR/inc/bootstrap while getopts bqs:t c; do case $c in b) RUN_TARGET=False ;; q) VERBOSE=False ;; s) STOP=$2 ;; t) RUN_BASE=False ;; esac done shift `expr $OPTIND - 1` # DIVIDER sudo mkdir -p $BASE_RELEASE_DIR $TARGET_RELEASE_DIR sudo chown -R `whoami` $STACK_ROOT # DIVIDER # DIVIDER if [[ -n "$LOGFILE" ]]; then LOGDAYS=${LOGDAYS:-7} TIMESTAMP_FORMAT=${TIMESTAMP_FORMAT:-"%F-%H%M%S"} CURRENT_LOG_TIME=$(date "+$TIMESTAMP_FORMAT") # DIVIDER LOGDIR=$(dirname "$LOGFILE") LOGNAME=$(basename "$LOGFILE") echo "Creating $LOGDIR...." sudo mkdir -p $LOGDIR sudo chown -R `whoami` $LOGDIR find $LOGDIR -maxdepth 1 -name $LOGNAME.\* -mtime +$LOGDAYS -exec rm {} \; LOGFILE=$LOGFILE.${CURRENT_LOG_TIME} SUMFILE=$LOGFILE.${CURRENT_LOG_TIME}.summary # DIVIDER exec 3>&1 if [[ "$VERBOSE" == "True" ]]; then echo "Running in verbose mode:" echo " Full logs found at => ${LOGFILE}" echo " Summary logs at => ${SUMFILE}" # DIVIDER exec 1> >( ./tools/outfilter.py -v -o "${LOGFILE}" ) 2>&1 # DIVIDER exec 6> >( ./tools/outfilter.py -o "${SUMFILE}" ) else echo "Running in summary mode:" echo " Full logs found at => ${LOGFILE}" echo " Summary logs at => ${SUMFILE}" # DIVIDER exec 1> >( ./tools/outfilter.py -o "${LOGFILE}") 2>&1 # DIVIDER exec 6> >( ./tools/outfilter.py -v -o "${SUMFILE}" >&3) fi echo_summary "grenade.sh log $LOGFILE" # DIVIDER ln -sf $LOGFILE $LOGDIR/$LOGNAME ln -sf $SUMFILE $LOGDIR/$LOGNAME.summary else # DIVIDER exec 3>&1 if [[ "$VERBOSE" != "yes" ]]; then # DIVIDER exec 1>/dev/null 2>&1 fi # DIVIDER exec 6> >( ./tools/outfilter.py -v -o "${SUMFILE}" >&3) fi # DIVIDER if [[ -n "$SCREEN_LOGDIR" ]]; then # DIVIDER if [[ -d "$SCREEN_LOGDIR" ]]; then # DIVIDER find $SCREEN_LOGDIR -maxdepth 1 -name screen-\*.log -mtime +$LOGDAYS -exec rm {} \; else sudo mkdir -p $SCREEN_LOGDIR sudo chown -R `whoami` $SCREEN_LOGDIR fi fi # DIVIDER set -o errexit # DIVIDER set -o xtrace # DIVIDER if [ "${GRENADE_USE_EXTERNAL_DEVSTACK}" != "True" ]; then # DIVIDER fetch_devstacks else devstacks_setup_environment fi # DIVIDER source $GRENADE_DIR/functions # DIVIDER export PS4='+ $(short_source): ' # DIVIDER export TOP_DIR=$TARGET_DEVSTACK_DIR # DIVIDER source $TARGET_DEVSTACK_DIR/inc/meta-config # DIVIDER extract_localrc_section $BASE_DEVSTACK_DIR/local.conf \ $BASE_DEVSTACK_DIR/localrc \ $BASE_DEVSTACK_DIR/.localrc.auto # DIVIDER ENABLED_SERVICES=$(set +o xtrace && source $BASE_DEVSTACK_DIR/stackrc && echo $ENABLED_SERVICES) # DIVIDER fetch_grenade_plugins # DIVIDER load_settings if [ "${GRENADE_USE_EXTERNAL_DEVSTACK}" != "True" ]; then # DIVIDER devstack_localrc base "CELLSV2_SETUP=singleconductor" devstack_localrc target "CELLSV2_SETUP=singleconductor" fi # DIVIDER extract_localrc_section $TARGET_DEVSTACK_DIR/local.conf \ $TARGET_DEVSTACK_DIR/localrc \ $TARGET_DEVSTACK_DIR/.localrc.auto # DIVIDER if [[ "$RUN_BASE" == "True" ]]; then # DIVIDER init_grenade_db if [ "${GRENADE_USE_EXTERNAL_DEVSTACK}" != "True" ]; then echo_summary "Running base stack.sh" cd $BASE_DEVSTACK_DIR GIT_BASE=$GIT_BASE ./stack.sh stop $STOP stack.sh 10 echo_summary "Running post-stack.sh" if [[ -e $GRENADE_DIR/post-stack.sh ]]; then cd $GRENADE_DIR # DIVIDER export SUB_NODE_ENV_VARS="CELLSV2_SETUP=singleconductor" sed -i 's/stdbuf/$SUB_NODE_ENV_VARS stdbuf/' ./post-stack.sh cat ./post-stack.sh ./post-stack.sh stop $STOP post-stack.sh 15 echo_summary "Completed post-stack.sh" fi fi # DIVIDER echo_summary "Caching downloaded images" mkdir -p $BASE_RELEASE_DIR/images echo "Images: $IMAGE_URLS" for image_url in ${IMAGE_URLS//,/ }; do IMAGE_FNAME=`basename "$image_url"` if [[ -r $BASE_DEVSTACK_DIR/files/$IMAGE_FNAME ]]; then rsync -a $BASE_DEVSTACK_DIR/files/$IMAGE_FNAME $BASE_RELEASE_DIR/images fi done # DIVIDER if [[ -d $BASE_DEVSTACK_DIR/files/images ]] ; then rsync -a $BASE_DEVSTACK_DIR/files/images/ $BASE_RELEASE_DIR/images fi stop $STOP image-cache 20 # DIVIDER # DIVIDER if [[ "$BASE_RUN_SMOKE" == "True" ]]; then echo_summary "Running base smoke test" cd $BASE_RELEASE_DIR/tempest # DIVIDER TEMPEST_VENV_UPPER_CONSTRAINTS=$(set +o xtrace && source $BASE_DEVSTACK_DIR/stackrc && echo $TEMPEST_VENV_UPPER_CONSTRAINTS) # DIVIDER if [[ "$TEMPEST_VENV_UPPER_CONSTRAINTS" != "master" ]]; then echo "Using $TEMPEST_VENV_UPPER_CONSTRAINTS constraints in Tempest virtual env." # DIVIDER export UPPER_CONSTRAINTS_FILE=$TEMPEST_VENV_UPPER_CONSTRAINTS export TOX_CONSTRAINTS_FILE=$TEMPEST_VENV_UPPER_CONSTRAINTS fi tox -esmoke -- --concurrency=$TEMPEST_CONCURRENCY if [ "${GRENADE_USE_EXTERNAL_DEVSTACK}" != "True" ]; then # DIVIDER if [[ -e $TARGET_RELEASE_DIR/tempest ]]; then rsync -a $BASE_RELEASE_DIR/tempest/.tox/ $TARGET_RELEASE_DIR/tempest/.tox/ if [[ -d $BASE_RELEASE_DIR/tempest/.testrepository ]]; then rsync -a $BASE_RELEASE_DIR/tempest/.testrepository/ $TARGET_RELEASE_DIR/tempest/.testrepository/ elif [[ -d $BASE_RELEASE_DIR/tempest/.stestr ]]; then rsync -a $BASE_RELEASE_DIR/tempest/.stestr/ $TARGET_RELEASE_DIR/tempest/.stestr/ fi fi fi fi stop $STOP base-smoke 110 # DIVIDER resources early_create # DIVIDER resources create # DIVIDER resources verify pre-upgrade # DIVIDER echo_summary "Saving current state information" $GRENADE_DIR/save-state stop $STOP save-state 130 # DIVIDER echo_summary "Shutting down all services on base devstack..." shutdown_services # DIVIDER resources verify_noapi pre-upgrade fi # DIVIDER if [[ "$RUN_TARGET" == "True" ]]; then # DIVIDER fetch_plugins # DIVIDER echo_summary "Preparing the target devstack environment" $GRENADE_DIR/prep-target # DIVIDER echo "Upgrade projects: $UPGRADE_PROJECTS" for project in $UPGRADE_PROJECTS; do echo "Upgrading project: $project" upgrade_service $project done # DIVIDER if [[ "$ENABLE_TEMPEST" == "True" ]]; then echo_summary "Running upgrade-tempest" $GRENADE_DIR/upgrade-tempest || die $LINENO "Failure in upgrade-tempest" stop $STOP upgrade-tempest 290 fi # DIVIDER # DIVIDER resources verify post-upgrade # DIVIDER if [[ "$TARGET_RUN_SMOKE" == "True" ]]; then echo_summary "Running tempest smoke tests" cd $TARGET_RELEASE_DIR/tempest tox -esmoke -- --concurrency=$TEMPEST_CONCURRENCY stop $STOP run-smoke 330 fi # DIVIDER save_data $TARGET_RELEASE $TARGET_DEVSTACK_DIR # DIVIDER resources destroy fi # DIVIDER echo_summary "Grenade has completed the pre-programmed upgrade scripts." # DIVIDER echo_summary "grenade.sh completed in $SECONDS seconds." # DIVIDER

Grenade assumes it is running on the system that will be hosting the upgrade processes

grenade.sh [-b] [-t] [-s stop-label] [-q]

-b Run only the base part -t Run only the target part (assumes a base run is in place) -q Quiet mode -s stop-label is the name of the step after which the script will stop. This is useful for debugging upgrades.

GRENADE_DIR is set once by the top level grenade.sh and exported so that all subsequent scripts can find their way back to the grenade root directory. No other scripts should set this variable

Source the bootstrapping facilities

Grenade attempts to reuse as much content from devstack on the target side as possible, but we need enough of our own code to get there.

grenaderc is a set of X=Y declarations that don't need any of the devstack functions to work

inc/bootstrap is the most minimal amount of functions that grenade needs to get going. This includes things like echo functions, trueorfalse, and the functions related to git cloning, so that we can get our devstack trees.

Create all the base directory structures needed for the rest of the environment to run.

This will give you an STACK_ROOT tree that you expect, and that your normal user owns for follow on activities.

Logging

TODO(sdague): should this extract into inc/bootstrap?

Set up logging Set LOGFILE to turn on logging Append '.xxxxxxxx' to the given name to maintain history where 'xxxxxxxx' is a representation of the date the file was created

First clean up old log files. Use the user-specified LOGFILE as the template to search for, appending '.*' to match the date we added on earlier runs.

Redirect output according to config Copy stdout to fd 3

Redirect stdout/stderr to tee to write the log file

Set up a second fd for output

Set fd 1 and 2 to primary logfile

Set fd 6 to summary logfile and stdout

Specified logfile name always links to the most recent log

Set up output redirection without log files Copy stdout to fd 3

Throw away stdout and stderr

Always send summary fd to original stdout

Set up logging of screen windows Set SCREEN_LOGDIR to turn on logging of screen windows to the directory specified in SCREEN_LOGDIR, we will log to the file screen-$SERVICE_NAME-$TIMESTAMP.log in that dir and have a link screen-$SERVICE_NAME.log to the latest log file. Logs are kept for as long specified in LOGDAYS.

We make sure the directory is created.

We cleanup the old logs

This script exits on an error so that errors don't compound and you see only the first error that occurred.

Print the commands being run so that we can see the command that triggers an error. It is also useful for following allowing as the install occurs.

Devstack Phase 2 initialization

We now have enough infrastructure in grenade.sh to go and get both SOURCE and TARGET devstack trees. After which point we 'pivot' onto the TARGET devstack functions file, then source the rest of the grenade settings. This should let us run the bulk of grenade.

Get both devstack trees, so that BASE_DEVSTACK_DIR, and TARGET_DEVSTACK_DIR are now fully populated.

Source the rest of the Grenade functions. For convenience $GRENADE_DIR/functions implicitly sources $TARGET_DEVSTACK_DIR/functions. So this line can't happen until we have the devstacks pulled down.

We now have the 'short_source' function available, so setup our PS4 variable

Many calls inside of devstack functions reference $TOP_DIR, which is the root of devstack. We export $TOP_DIR to all child processes here to be the TARGET_DEVSTACK_DIR.

If you want a script to use functions off of BASE_DEVSTACK_DIR (like the shutdown phase) you must explicitly reset TOP_DIR in those scripts.

Install 'Base' Build of OpenStack

Oh the complexity of bootstrapping. We need to populate enabled services from what devstack-gate might have set it as.

Collect the ENABLED_SERVICES from the base directory, this is what we are starting with.

Fetch all the grenade plugins which were registered in pluginrc via the enable_grenade_plugin stanza. This must be done before settings are loaded, but has to be this late to give access to all the devstack functions.

Load the settings files for all the in tree projects/. This registers all the projects in order that we're going to be upgrading when the time is right.

Nova should use singleconductor as Grenade doesn't setup multi-cell rabbit for now

And ensure that we setup the target localrc.auto, because stack.sh isn't run there. This has to be run after load_settings because plugins might change the service list during this phase.

Run the base install of the environment

Initialize grenade_db local storage, used for resource tracking

By the time we get here the sub nodes are already setup with localrc files as those are transferred in devstack-gate even before grenade.sh is called We hack the ./post-stack.sh to inject what we need. if we don't set CELLSV2_SETUP, the default devstack assumes "superconductor" and fails.

Cache downloaded instances

NOTE(sileht): If glance is not enabled the directory cannot exists.

Operation

Validate the install

NOTE(yoctozepto): Grenade does not know about TEMPEST_VENV_UPPER_CONSTRAINTS, only DevStack does. This sources that one variable from it.

NOTE(gmann): If gate explicitly set the non master constraints to use for Tempest venv then use the same while running the tests too otherwise, it will recreate the Tempest venv due to constraints mismatch. recreation of Tempest venv can flush the initially installed tempest plugins and their deps.

NOTE: setting both tox env var and once Tempest start using new var TOX_CONSTRAINTS_FILE then we can remove the old one.

once we are done, copy our created artifacts to the target

Early Create resources, used largely for network setup

Create resources

Verify the resources were created

Save some stuff before we shut that whole thing down

Shut down running code

Verify the resources still exist after the shutdown

Upgrades

Clone all devstack plugins on the new side, because we're not running a full stack.sh

Get target devstack tree ready for services to be run from it, including trying to reuse any existing files we pulled during the base run.

upgrade all the projects in order

Upgrade Tempest

Upgrade Tests

Verify the resources still exist after the upgrade

Validate the upgrade This is used for testing grenade locally, and should not be used in the gate. Instead, grenade.sh runs smoke tests on the old cloud above, but smoke tests are run on the upgraded cloud by the gate script after grenade.sh has finished. If this option is enabled in the gate, tempest will be run twice against the new cloud.

Save databases

Cleanup the resources

Fin

Indicate how long this took to run (bash maintained variable SECONDS)