From 458a2d31b10701370746f3029c5bc580e85f24d9 Mon Sep 17 00:00:00 2001 From: Luciano Resende Date: Thu, 26 Jan 2017 15:19:15 -0800 Subject: Add utility scripts to help with initializing profiles --- bin/config | 35 +++++++++++++++++++++++ bin/init.sh | 93 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bin/start.sh | 34 ++++++++++++++++++++++ 3 files changed, 162 insertions(+) create mode 100644 bin/config create mode 100644 bin/init.sh create mode 100644 bin/start.sh diff --git a/bin/config b/bin/config new file mode 100644 index 0000000..2befb06 --- /dev/null +++ b/bin/config @@ -0,0 +1,35 @@ +# +# (C) Copyright IBM Corp. 2017 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Update these values to configure Toree runtime characteristics. +TOREE_GATEWAY_HOME="/opt/toree-gateway" +GLOBAL_FIRST_PORT_NUMBER="44000" +GLOBAL_NUMBER_OF_PORTS="18" # Should be a multiple of 5 + +# Settings that are global to all kernel instances. This is basically +# everything but the port settings. +TOREE_IP=${TOREE_IP:="9.30.137.220"} +TOREE_TRANSPORT=${TOREE_TRANSPORT:="tcp"} +TOREE_SIG_SCHEME=${TOREE_SIG_SCHEME:="hmac-sha256"} +# TOREE_KEY=${TOREE_KEY:="11111111-2222-3333-4444-555555555555"} + +SPARK_HOST=${SPARK_HOST:="STLAB18.SVL.IBM.COM"} +SPARK_PORT=${SPARK_PORT:="7077"} +SPARK_CPUS=${SPARK_CPUS:="2"} +SPARK_MEM=${SPARK_MEM:="1g"} +# SPARK_SECRET=${SPARK_SECRET:="your_secret_here"} + +KERNEL_LOGLEVEL=${KERNEL_LOGLEVEL:="DEBUG"} diff --git a/bin/init.sh b/bin/init.sh new file mode 100644 index 0000000..836773b --- /dev/null +++ b/bin/init.sh @@ -0,0 +1,93 @@ +# +# (C) Copyright IBM Corp. 2017 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Create the specified directory. +create_dir(){ + echo "Creating $1 ..." + mkdir -p $1 + status=$? + if [ $status -ne 0 ]; then + echo "Failed to create directory $1 ($status)" + exit $status + fi +} + +# Build the connection profiles that the kernel will use. +build_connection_profiles(){ + let n=$1 + let p=$GLOBAL_FIRST_PORT_NUMBER + + for ((k=1; k<=n; k++)) + do + # echo "--- Initialize kernel instance $k" + KERNEL_STATE="kernel-$k" + create_dir $TOREE_GATEWAY_HOME/profiles/$KERNEL_STATE + PROFILE="$TOREE_GATEWAY_HOME/profiles/$KERNEL_STATE/profile.json" + echo "{" > $PROFILE + echo " \"stdin_port\": $p," >> $PROFILE; let p=p+1 + echo " \"control_port\": $p," >> $PROFILE; let p=p+1 + echo " \"hb_port\": $p," >> $PROFILE; let p=p+1 + echo " \"shell_port\": $p," >> $PROFILE; let p=p+1 + echo " \"iopub_port\": $p," >> $PROFILE; let p=p+1 + echo " \"ip\": \"$TOREE_IP\"," >> $PROFILE + echo " \"transport\": \"$TOREE_TRANSPORT\"," >> $PROFILE + echo " \"signature_scheme\": \"$TOREE_SIG_SCHEME\"," >> $PROFILE + # echo " \"key\": \"$TOREE_KEY\"" >> $PROFILE + echo " \"key\": \"\"," >> $PROFILE + echo " \"py4j_java\": $p," >> $PROFILE; let p=p+500 + echo " \"py4j_python\": $p" >> $PROFILE; let p=p+1 + echo "}" >> $PROFILE + let p=p-501 + done +} + +# Calculate the number of kernels we can initialize, given the number of ports provided. +# $1 == number of ports available to use +# sets num_kernels == number of kernels (global variable, side effect, I know ...) +get_num_kernels() { + let m=$1%5 + if [ $m -ne 0 ]; then + echo "Info: $1 is not a multiple of 5." + echo " $m ports will not be used." + fi + + let num_kernels=$1/5 + echo "Number of kernels: $num_kernels" +} + +# Source the specified config file. +# $1 == specified config file +# $2 == default config file +source_config(){ + if [ ! -z "$1" ]; then + if [ ! -f $1 ]; then + echo "File $1 not found!" + exit 2 + else + source $1 + fi + else + source ../conf/config + fi +} + +# Get the installation configuation settings. +source_config + +# Get the number of kernels and build the connection profiles for all of +# the kernel instances. +get_num_kernels $GLOBAL_NUMBER_OF_PORTS # sets num_kernels +build_connection_profiles $num_kernels diff --git a/bin/start.sh b/bin/start.sh new file mode 100644 index 0000000..0514bba --- /dev/null +++ b/bin/start.sh @@ -0,0 +1,34 @@ +# +# (C) Copyright IBM Corp. 2017 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +str="'$*'" + +timestamp() { + date +"%Y-%m-%d_%H-%M-%S" +} + +echo "$str" > ../log/kernel_$(timestamp)_params.log + +# Start the kernel as a background process. +nohup ./run.sh "$@" > ../log/kernel_$(timestamp).log & + +# Acquire the pid for the kernel just started and write it to the pid file ... +echo $! + +# Return with a status in $?, and output the kernel slot as the handle that +# the caller provides when stopping the kernel. +# echo "$free_handle" +exit 0 \ No newline at end of file -- cgit v1.2.3