For those interested in the Discourse launcher bash script
Ref: discourse_docker/launcher at master · discourse/discourse_docker · GitHub
cat /var/discourse/launcher
#!/usr/bin/env bash
usage () {
echo "Usage: launcher COMMAND CONFIG [--skip-prereqs] [--docker-args STRING]"
echo "Commands:"
echo " start: Start/initialize a container"
echo " stop: Stop a running container"
echo " restart: Restart a container"
echo " destroy: Stop and remove a container"
echo " enter: Open a shell to run commands inside the container"
echo " logs: View the Docker logs for a container"
echo " bootstrap: Bootstrap a container for the config based on a template"
echo " run: Run the given command with the config in the context of the last bootstrapped image"
echo " rebuild: Rebuild a container (destroy old, bootstrap, start new)"
echo " cleanup: Remove all containers that have stopped for > 24 hours"
echo " start-cmd: Generate docker command used to start container"
echo
echo "Options:"
echo " --skip-prereqs Don't check launcher prerequisites"
echo " --docker-args Extra arguments to pass when running docker"
echo " --skip-mac-address Don't assign a mac address"
echo " --run-image Override the image used for running the container"
exit 1
}
command=$1
config=$2
# user_args_argv is assigned once when the argument vector is parsed.
user_args_argv=""
# user_args is mutable: its value may change when templates are parsed.
# Superset of user_args_argv.
user_args=""
user_run_image=""
if [[ $command == "run" ]]; then
run_command=$3
fi
while [ ${#} -gt 0 ]; do
case "${1}" in
--debug)
DEBUG="1"
;;
--skip-prereqs)
SKIP_PREREQS="1"
;;
--skip-mac-address)
SKIP_MAC_ADDRESS="1"
;;
--docker-args)
user_args_argv="$2"
user_args="$user_args_argv"
shift
;;
--run-image)
user_run_image="$2"
shift
;;
esac
shift 1
done
if [ -z "$command" -o -z "$config" -a "$command" != "cleanup" ]; then
usage
exit 1
fi
# Docker doesn't like uppercase characters, spaces or special characters, catch it now before we build everything out and then find out
re='[[:upper:]/ !@#$%^&*()+~`=]'
if [[ $config =~ $re ]];
then
echo
echo "ERROR: Config name '$config' must not contain upper case characters, spaces or special characters. Correct config name and rerun $0."
echo
exit 1
fi
cd "$(dirname "$0")"
docker_min_version='17.03.1'
docker_rec_version='17.06.2'
git_min_version='1.8.0'
git_rec_version='1.8.0'
config_file=containers/"$config".yml
cidbootstrap=cids/"$config"_bootstrap.cid
local_discourse=local_discourse
image="discourse/base:2.0.20200512-1735"
docker_path=`which docker.io 2> /dev/null || which docker`
git_path=`which git`
if [ "${SUPERVISED}" = "true" ]; then
restart_policy="--restart=no"
attach_on_start="-a"
attach_on_run="-a stdout -a stderr"
else
attach_on_run="-d"
fi
if [ -n "$DOCKER_HOST" ]; then
docker_ip=`sed -e 's/^tcp:\/\/\(.*\):.*$/\1/' <<< "$DOCKER_HOST"`
elif [ -x "$(which ip 2>/dev/null)" ]; then
docker_ip=`ip addr show docker0 | \
grep 'inet ' | \
awk '{ split($2,a,"/"); print a[1] }';`
else
docker_ip=`ifconfig | \
grep -B1 "inet addr" | \
awk '{ if ( $1 == "inet" ) { print $2 } else if ( $2 == "Link" ) { printf "%s:" ,$1 } }' | \
grep docker0 | \
awk -F: '{ print $3 }';`
fi
# From https://stackoverflow.com/a/44660519/702738
compare_version() {
if [[ $1 == $2 ]]; then
return 1
fi
local IFS=.
local i a=(${1%%[^0-9.]*}) b=(${2%%[^0-9.]*})
local arem=${1#${1%%[^0-9.]*}} brem=${2#${2%%[^0-9.]*}}
for ((i=0; i<${#a[@]} || i<${#b[@]}; i++)); do
if ((10#${a[i]:-0} < 10#${b[i]:-0})); then
return 1
elif ((10#${a[i]:-0} > 10#${b[i]:-0})); then
return 0
fi
done
if [ "$arem" '<' "$brem" ]; then
return 1
elif [ "$arem" '>' "$brem" ]; then
return 0
fi
return 1
}
install_docker() {
echo "Docker is not installed, you will need to install Docker in order to run Launcher"
echo "See https://docs.docker.com/installation/"
exit 1
}
check_prereqs() {
if [ -z $docker_path ]; then
install_docker
fi
# 1. docker daemon running?
# we send stderr to /dev/null cause we don't care about warnings,
# it usually complains about swap which does not matter
test=`$docker_path info 2> /dev/null`
if [[ $? -ne 0 ]] ; then
echo "Cannot connect to the docker daemon - verify it is running and you have access"
exit 1
fi
# 2. running an approved storage driver?
if ! $docker_path info 2> /dev/null | egrep -q 'Storage Driver: (aufs|zfs|overlay2)$'; then
echo "Your Docker installation is not using a supported storage driver. If we were to proceed you may have a broken install."
echo "aufs is the recommended storage driver, although zfs and overlay2 may work as well."
echo "Other storage drivers are known to be problematic."
echo "You can tell what filesystem you are using by running \"docker info\" and looking at the 'Storage Driver' line."
echo
echo "If you wish to continue anyway using your existing unsupported storage driver,"
echo "read the source code of launcher and figure out how to bypass this check."
exit 1
fi
# 3. running recommended docker version
test=($($docker_path --version)) # Get docker version string
test=${test[2]//,/} # Get version alone and strip comma if exists
# At least minimum docker version
if compare_version "${docker_min_version}" "${test}"; then
echo "ERROR: Docker version ${test} not supported, please upgrade to at least ${docker_min_version}, or recommended ${docker_rec_version}"
exit 1
fi
# Recommend newer docker version
if compare_version "${docker_rec_version}" "${test}"; then
echo "WARNING: Docker version ${test} deprecated, recommend upgrade to ${docker_rec_version} or newer."
fi
# 4. discourse docker image is downloaded
test=`$docker_path images | awk '{print $1 ":" $2 }' | grep "$image"`
if [ -z "$test" ]; then
echo
echo "WARNING: We are about to start downloading the Discourse base image"
echo "This process may take anywhere between a few minutes to an hour, depending on your network speed"
echo
echo "Please be patient"
echo
fi
# 5. running recommended git version
test=($($git_path --version)) # Get git version string
test=${test[2]//,/} # Get version alone and strip comma if exists
# At least minimum version
if compare_version "${git_min_version}" "${test}"; then
echo "ERROR: Git version ${test} not supported, please upgrade to at least ${git_min_version}, or recommended ${git_rec_version}"
exit 1
fi
# Recommend best version
if compare_version "${git_rec_version}" "${test}"; then
echo "WARNING: Git version ${test} deprecated, recommend upgrade to ${git_rec_version} or newer."
fi
# 6. able to attach stderr / out / tty
test=`$docker_path run $user_args -i --rm -a stdout -a stderr $image echo working`
if [[ "$test" =~ "working" ]] ; then : ; else
echo "Your Docker installation is not working correctly"
echo
echo "See: https://meta.discourse.org/t/docker-error-on-bootstrap/13657/18?u=sam"
exit 1
fi
# 7. enough space for the bootstrap on docker folder
folder=`$docker_path info --format '{{.DockerRootDir}}'`
safe_folder=${folder:-/var/lib/docker}
test=$(($(stat -f --format="%a*%S" $safe_folder)/1024**3 < 5))
if [[ $test -ne 0 ]] ; then
echo "You have less than 5GB of free space on the disk where $safe_folder is located. You will need more space to continue"
df -h $safe_folder
echo
if tty >/dev/null; then
read -p "Would you like to attempt to recover space by cleaning docker images and containers in the system?(y/N)" -n 1 -r
echo
if [[ $REPLY =~ ^[Yy]$ ]]
then
$docker_path container prune --force --filter until=1h >/dev/null
$docker_path image prune --all --force --filter until=1h >/dev/null
echo "If the cleanup was successful, you may try again now"
fi
fi
exit 1
fi
}
if [ -z "$SKIP_PREREQS" ] && [ "$command" != "cleanup" ]; then
check_prereqs
fi
host_run() {
read -r -d '' env_ruby << 'RUBY'
require 'yaml'
input = STDIN.readlines.join
yaml = YAML.load(input)
if host_run = yaml['host_run']
params = yaml['params'] || {}
host_run.each do |run|
params.each do |k,v|
run = run.gsub("$#{k}", v)
end
STDOUT.write "#{run}--SEP--"
end
end
RUBY
host_run=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e "$env_ruby"`
while [ "$host_run" ] ; do
iter=${host_run%%--SEP--*}
echo
echo "Host run: $iter"
$iter || exit 1
echo
host_run="${host_run#*--SEP--}"
done
}
set_volumes() {
volumes=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
"require 'yaml'; puts YAML.load(STDIN.readlines.join)['volumes'].map{|v| '-v ' << v['volume']['host'] << ':' << v['volume']['guest'] << ' '}.join"`
}
set_links() {
links=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
"require 'yaml'; puts YAML.load(STDIN.readlines.join)['links'].map{|l| '--link ' << l['link']['name'] << ':' << l['link']['alias'] << ' '}.join"`
}
find_templates() {
local templates=`cat $1 | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
"require 'yaml'; puts YAML.load(STDIN.readlines.join)['templates']"`
local arrTemplates=${templates// / }
if [ ! -z "$templates" ]; then
for template in "${arrTemplates[@]}"
do
local nested_templates=$(find_templates $template)
if [ ! -z "$nested_templates" ]; then
templates="$templates $nested_templates"
fi
done
echo $templates
else
echo ""
fi
}
set_template_info() {
templates=$(find_templates $config_file)
arrTemplates=(${templates// / })
config_data=$(cat $config_file)
input="hack: true"
for template in "${arrTemplates[@]}"
do
[ ! -z $template ] && {
input="$input _FILE_SEPERATOR_ $(cat $template)"
}
done
# we always want our config file last so it takes priority
input="$input _FILE_SEPERATOR_ $config_data"
read -r -d '' env_ruby << 'RUBY'
require 'yaml'
input=STDIN.readlines.join
# default to UTF-8 for the dbs sake
env = {'LANG' => 'en_US.UTF-8'}
input.split('_FILE_SEPERATOR_').each do |yml|
yml.strip!
begin
env.merge!(YAML.load(yml)['env'] || {})
rescue Psych::SyntaxError => e
puts e
puts "*ERROR."
rescue => e
puts yml
p e
end
end
puts env.map{|k,v| "-e\n#{k}=#{v}" }.join("\n")
RUBY
tmp_input_file=$(mktemp)
echo "$input" > "$tmp_input_file"
raw=`exec cat "$tmp_input_file" | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e "$env_ruby"`
rm -f "$tmp_input_file"
env=()
ok=1
while read i; do
if [ "$i" == "*ERROR." ]; then
ok=0
elif [ -n "$i" ]; then
env[${#env[@]}]="${i//\{\{config\}\}/${config}}"
fi
done <<< "$raw"
if [ "$ok" -ne 1 ]; then
echo "${env[@]}"
echo "YAML syntax error. Please check your containers/*.yml config files."
exit 1
fi
# labels
read -r -d '' labels_ruby << 'RUBY'
require 'yaml'
input=STDIN.readlines.join
labels = {}
input.split('_FILE_SEPERATOR_').each do |yml|
yml.strip!
begin
labels.merge!(YAML.load(yml)['labels'] || {})
rescue Psych::SyntaxError => e
puts e
puts "*ERROR."
rescue => e
puts yml
p e
end
end
puts labels.map{|k,v| "-l\n#{k}=#{v}" }.join("\n")
RUBY
tmp_input_file=$(mktemp)
echo "$input" > "$tmp_input_file"
raw=`exec cat "$tmp_input_file" | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e "$labels_ruby"`
rm -f "$tmp_input_file"
labels=()
ok=1
while read i; do
if [ "$i" == "*ERROR." ]; then
ok=0
elif [ -n "$i" ]; then
labels[${#labels[@]}]=$(echo $i | sed s/{{config}}/${config}/g)
fi
done <<< "$raw"
if [ "$ok" -ne 1 ]; then
echo "${labels[@]}"
echo "YAML syntax error. Please check your containers/*.yml config files."
exit 1
fi
# expose
read -r -d '' ports_ruby << 'RUBY'
require 'yaml'
input=STDIN.readlines.join
ports = []
input.split('_FILE_SEPERATOR_').each do |yml|
yml.strip!
begin
ports += (YAML.load(yml)['expose'] || [])
rescue Psych::SyntaxError => e
puts e
puts "*ERROR."
rescue => e
puts yml
p e
end
end
puts ports.map { |p| p.to_s.include?(':') ? "-p\n#{p}" : "--expose\n#{p}" }.join("\n")
RUBY
tmp_input_file=$(mktemp)
echo "$input" > "$tmp_input_file"
raw=`exec cat "$tmp_input_file" | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e "$ports_ruby"`
rm -f "$tmp_input_file"
ports=()
ok=1
while read i; do
if [ "$i" == "*ERROR." ]; then
ok=0
elif [ -n "$i" ]; then
ports[${#ports[@]}]=$i
fi
done <<< "$raw"
if [ "$ok" -ne 1 ]; then
echo "${ports[@]}"
echo "YAML syntax error. Please check your containers/*.yml config files."
exit 1
fi
merge_user_args
}
if [ -z $docker_path ]; then
install_docker
fi
[ "$command" == "cleanup" ] && {
$docker_path container prune --filter until=1h
$docker_path image prune --all --filter until=1h
if [ -d /var/discourse/shared/standalone/postgres_data_old ]; then
echo
echo "Old PostgreSQL backup data cluster detected taking up $(du -hs /var/discourse/shared/standalone/postgres_data_old | awk '{print $1}') detected"
read -p "Would you like to remove it? (Y/n): " -n 1 -r && echo
if [[ $REPLY =~ ^[Yy]$ ]]; then
echo "removing old PostgreSQL data cluster at /var/discourse/shared/standalone/postgres_data_old..."
rm -rf /var/discourse/shared/standalone/postgres_data_old
else
exit 1
fi
fi
exit 0
}
if [ ! "$command" == "setup" ]; then
if [[ ! -e $config_file ]]; then
echo "Config file was not found, ensure $config_file exists"
echo
echo "Available configs ( `cd containers && ls -dm *.yml | tr -s '\n' ' ' | awk '{ gsub(/\.yml/, ""); print }'`)"
exit 1
fi
fi
docker_version=($($docker_path --version))
docker_version=${test[2]//,/}
restart_policy=${restart_policy:---restart=always}
set_existing_container(){
existing=`$docker_path ps -a | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
}
run_stop() {
set_existing_container
if [ ! -z $existing ]
then
(
set -x
$docker_path stop -t 10 $config
)
else
echo "$config was not started !"
echo "./discourse-doctor may help diagnose the problem."
exit 1
fi
}
set_run_image() {
run_image=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
"require 'yaml'; puts YAML.load(STDIN.readlines.join)['run_image']"`
if [ -n "$user_run_image" ]; then
run_image=$user_run_image
elif [ -z "$run_image" ]; then
run_image="$local_discourse/$config"
fi
}
set_boot_command() {
boot_command=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
"require 'yaml'; puts YAML.load(STDIN.readlines.join)['boot_command']"`
if [ -z "$boot_command" ]; then
no_boot_command=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
"require 'yaml'; puts YAML.load(STDIN.readlines.join)['no_boot_command']"`
if [ -z "$no_boot_command" ]; then
boot_command="/sbin/boot"
fi
fi
}
merge_user_args() {
local docker_args
docker_args=`cat $config_file | $docker_path run $user_args --rm -i -a stdout -a stdin $image ruby -e \
"require 'yaml'; puts YAML.load(STDIN.readlines.join)['docker_args']"`
if [[ -n "$docker_args" ]]; then
user_args="$user_args_argv $docker_args"
fi
}
run_start() {
if [ -z "$START_CMD_ONLY" ]
then
existing=`$docker_path ps | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
echo $existing
if [ ! -z $existing ]
then
echo "Nothing to do, your container has already started!"
exit 0
fi
existing=`$docker_path ps -a | awk '{ print $1, $(NF) }' | grep " $config$" | awk '{ print $1 }'`
if [ ! -z $existing ]
then
echo "starting up existing container"
(
set -x
$docker_path start $config
)
exit 0
fi
fi
host_run
set_template_info
set_volumes
set_links
set_run_image
set_boot_command
# get hostname and settings from container configuration
for envar in "${env[@]}"
do
if [[ $envar == DOCKER_USE_HOSTNAME* ]] || [[ $envar == DISCOURSE_HOSTNAME* ]]
then
# use as environment variable
eval $envar
fi
done
(
hostname=`hostname -s`
# overwrite hostname
if [ "$DOCKER_USE_HOSTNAME" = "true" ]
then
hostname=$DISCOURSE_HOSTNAME
else
hostname=$hostname-$config
fi
# we got to normalize so we only have allowed strings, this is more comprehensive but lets see how bash does first
# hostname=`$docker_path run $user_args --rm $image ruby -e 'print ARGV[0].gsub(/[^a-zA-Z-]/, "-")' $hostname`
# docker added more hostname rules
hostname=${hostname//_/-}
if [ -z "$SKIP_MAC_ADDRESS" ] ; then
mac_address="--mac-address $($docker_path run $user_args -i --rm -a stdout -a stderr $image /bin/sh -c "echo $hostname | md5sum | sed 's/^\(..\)\(..\)\(..\)\(..\)\(..\).*$/02:\1:\2:\3:\4:\5/'")"
fi
if [ ! -z "$START_CMD_ONLY" ] ; then
docker_path="true"
fi
set -x
$docker_path run --shm-size=512m $links $attach_on_run $restart_policy "${env[@]}" "${labels[@]}" -h "$hostname" \
-e DOCKER_HOST_IP="$docker_ip" --name $config -t "${ports[@]}" $volumes $mac_address $user_args \
$run_image $boot_command
)
exit 0
}
run_run() {
set_template_info
set_volumes
set_links
set_run_image
unset ERR
(exec $docker_path run --rm --shm-size=512m $user_args $links "${env[@]}" -e DOCKER_HOST_IP="$docker_ip" -i -a stdin -a stdout -a stderr $volumes $run_image \
/bin/bash -c "$run_command") || ERR=$?
if [[ $ERR > 0 ]]; then
exit 1
fi
}
run_bootstrap() {
host_run
# Is the image available?
# If not, pull it here so the user is aware what's happening.
$docker_path history $image >/dev/null 2>&1 || $docker_path pull $image
set_template_info
base_image=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
"require 'yaml'; puts YAML.load(STDIN.readlines.join)['base_image']"`
update_pups=`cat $config_file | $docker_path run $user_args --rm -i -a stdin -a stdout $image ruby -e \
"require 'yaml'; puts YAML.load(STDIN.readlines.join)['update_pups']"`
if [[ ! X"" = X"$base_image" ]]; then
image=$base_image
fi
set_volumes
set_links
rm -f $cidbootstrap
run_command="cd /pups &&"
if [[ ! "false" = $update_pups ]]; then
run_command="$run_command git pull &&"
fi
run_command="$run_command /pups/bin/pups --stdin"
echo $run_command
unset ERR
tmp_input_file=$(mktemp)
echo "$input" > "$tmp_input_file"
(exec cat "$tmp_input_file" | $docker_path run --shm-size=512m $user_args $links "${env[@]}" -e DOCKER_HOST_IP="$docker_ip" --cidfile $cidbootstrap -i -a stdin -a stdout -a stderr $volumes $image \
/bin/bash -c "$run_command") || ERR=$?
rm -f "$tmp_input_file"
unset FAILED
# magic exit code that indicates a retry
if [[ "$ERR" == 77 ]]; then
$docker_path rm `cat $cidbootstrap`
rm $cidbootstrap
exit 77
elif [[ "$ERR" > 0 ]]; then
FAILED=TRUE
fi
if [[ $FAILED = "TRUE" ]]; then
if [[ ! -z "$DEBUG" ]]; then
$docker_path commit `cat $cidbootstrap` $local_discourse/$config-debug || echo 'FAILED TO COMMIT'
echo "** DEBUG ** Maintaining image for diagnostics $local_discourse/$config-debug"
fi
$docker_path rm `cat $cidbootstrap`
rm $cidbootstrap
echo "** FAILED TO BOOTSTRAP ** please scroll up and look for earlier error messages, there may be more than one."
echo "./discourse-doctor may help diagnose the problem."
exit 1
fi
sleep 5
$docker_path commit `cat $cidbootstrap` $local_discourse/$config || echo 'FAILED TO COMMIT'
$docker_path rm `cat $cidbootstrap` && rm $cidbootstrap
}
case "$command" in
bootstrap)
run_bootstrap
echo "Successfully bootstrapped, to startup use ./launcher start $config"
exit 0
;;
run)
run_run
exit 0
;;
enter)
exec $docker_path exec -it $config /bin/bash --login
;;
stop)
run_stop
exit 0
;;
logs)
$docker_path logs $config
exit 0
;;
restart)
run_stop
run_start
exit 0
;;
start-cmd)
START_CMD_ONLY="1"
run_start
exit 0;
;;
start)
run_start
exit 0
;;
rebuild)
if [ "$(git symbolic-ref --short HEAD)" == "master" ]; then
echo "Ensuring launcher is up to date"
git remote update
LOCAL=$(git rev-parse HEAD)
REMOTE=$(git rev-parse @{u})
BASE=$(git merge-base HEAD @{u})
if [ $LOCAL = $REMOTE ]; then
echo "Launcher is up-to-date"
elif [ $LOCAL = $BASE ]; then
echo "Updating Launcher"
git pull || (echo 'failed to update' && exit 1)
for (( i=${#BASH_ARGV[@]}-1,j=0; i>=0,j<${#BASH_ARGV[@]}; i--,j++ ))
do
args[$j]=${BASH_ARGV[$i]}
done
exec bash $0 "${args[@]}" # $@ is empty, because of shift at the beginning. Use BASH_ARGV instead.
elif [ $REMOTE = $BASE ]; then
echo "Your version of Launcher is ahead of origin"
else
echo "Launcher has diverged source, this is only expected in Dev mode"
fi
fi
set_existing_container
if [ ! -z $existing ]
then
echo "Stopping old container"
(
set -x
$docker_path stop -t 60 $config
)
fi
run_bootstrap
if [ ! -z $existing ]
then
echo "Removing old container"
(
set -x
$docker_path rm $config
)
fi
run_start
exit 0
;;
destroy)
(set -x; $docker_path stop -t 10 $config && $docker_path rm $config) || (echo "$config was not found" && exit 0)
exit 0
;;
esac
usage
Any thoughts or ideas?
See also: