Skip to content

Kotlin spark interpreter windows fixes #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ileasile wants to merge 1 commit into
base: kotlin-spark-interpreter
Choose a base branch
from
Open

Kotlin spark interpreter windows fixes #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
147 changes: 147 additions & 0 deletions bin/common-win.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

if [ -L ${BASH_SOURCE-$0} ]; then
FWDIR=$(dirname $(readlink "${BASH_SOURCE-$0}"))
else
FWDIR=$(dirname "${BASH_SOURCE-$0}")
fi

if [[ -z "${ZEPPELIN_HOME}" ]]; then
# Make ZEPPELIN_HOME look cleaner in logs by getting rid of the
# extra ../
export ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)"
export ZEPPELIN_HOME="${ZEPPELIN_HOME/\/c\//C:\/}"
fi

if [[ -z "${ZEPPELIN_CONF_DIR}" ]]; then
export ZEPPELIN_CONF_DIR="${ZEPPELIN_HOME}/conf"
fi

if [[ -z "${ZEPPELIN_LOG_DIR}" ]]; then
export ZEPPELIN_LOG_DIR="${ZEPPELIN_HOME}/logs"
fi

if [[ -z "$ZEPPELIN_PID_DIR" ]]; then
export ZEPPELIN_PID_DIR="${ZEPPELIN_HOME}/run"
fi

if [[ -z "${ZEPPELIN_WAR}" ]]; then
if [[ -d "${ZEPPELIN_HOME}/zeppelin-web/dist" ]]; then
export ZEPPELIN_WAR="${ZEPPELIN_HOME}/zeppelin-web/dist"
else
export ZEPPELIN_WAR=$(find -L "${ZEPPELIN_HOME}" -name "zeppelin-web*.war")
fi
fi

if [[ -f "${ZEPPELIN_CONF_DIR}/zeppelin-env.sh" ]]; then
. "${ZEPPELIN_CONF_DIR}/zeppelin-env.sh"
fi

ZEPPELIN_CLASSPATH+=";${ZEPPELIN_CONF_DIR}"

function addEachJarInDir(){
if [[ -d "${1}" ]]; then
for jar in $(find -L "${1}" -maxdepth 1 -name '*jar'); do
ZEPPELIN_CLASSPATH="$jar;$ZEPPELIN_CLASSPATH"
done
fi
}

function addEachJarInDirRecursive(){
if [[ -d "${1}" ]]; then
for jar in $(find -L "${1}" -type f -name '*jar'); do
ZEPPELIN_CLASSPATH="$jar;$ZEPPELIN_CLASSPATH"
done
fi
}

function addEachJarInDirRecursiveForIntp(){
if [[ -d "${1}" ]]; then
for jar in $(find -L "${1}" -type f -name '*jar'); do
ZEPPELIN_INTP_CLASSPATH="$jar;$ZEPPELIN_INTP_CLASSPATH"
done
fi
}

function addJarInDir(){
if [[ -d "${1}" ]]; then
ZEPPELIN_CLASSPATH="${1}/*;${ZEPPELIN_CLASSPATH}"
fi
}

function addJarInDirForIntp() {
if [[ -d "${1}" ]]; then
ZEPPELIN_INTP_CLASSPATH="${1}/*;${ZEPPELIN_INTP_CLASSPATH}"
fi
}

ZEPPELIN_COMMANDLINE_MAIN=org.apache.zeppelin.utils.CommandLineUtils

function getZeppelinVersion(){
if [[ -d "${ZEPPELIN_HOME}/zeppelin-server/target/classes" ]]; then
ZEPPELIN_CLASSPATH+=";${ZEPPELIN_HOME}/zeppelin-server/target/classes"
fi
addJarInDir "${ZEPPELIN_HOME}/zeppelin-server/target/lib"
CLASSPATH+=";${ZEPPELIN_CLASSPATH}"
$ZEPPELIN_RUNNER -cp $CLASSPATH $ZEPPELIN_COMMANDLINE_MAIN -v
exit 0
}

# Text encoding for
# read/write job into files,
# receiving/displaying query/result.
if [[ -z "${ZEPPELIN_ENCODING}" ]]; then
export ZEPPELIN_ENCODING="UTF-8"
fi

if [[ -z "${ZEPPELIN_MEM}" ]]; then
export ZEPPELIN_MEM="-Xms1024m -Xmx1024m -XX:MaxPermSize=512m"
fi

if [[ -z "${ZEPPELIN_INTP_MEM}" ]]; then
export ZEPPELIN_INTP_MEM="-Xms1024m -Xmx1024m -XX:MaxPermSize=512m"
fi

JAVA_OPTS+=" ${ZEPPELIN_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING} ${ZEPPELIN_MEM}"
JAVA_OPTS+=" -Dlog4j.configuration=file:///${ZEPPELIN_CONF_DIR}/log4j.properties"
export JAVA_OPTS

JAVA_INTP_OPTS="${ZEPPELIN_INTP_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING}"
if [[ -z "${ZEPPELIN_SPARK_YARN_CLUSTER}" ]]; then
JAVA_INTP_OPTS+=" -Dlog4j.configuration=file:///${ZEPPELIN_CONF_DIR}/log4j.properties"
else
JAVA_INTP_OPTS+=" -Dlog4j.configuration=log4j_yarn_cluster.properties"
fi
export JAVA_INTP_OPTS


if [[ -n "${JAVA_HOME}" ]]; then
ZEPPELIN_RUNNER="${JAVA_HOME}\\bin\\java"
else
ZEPPELIN_RUNNER=java
fi
export ZEPPELIN_RUNNER

if [[ -z "$ZEPPELIN_IDENT_STRING" ]]; then
export ZEPPELIN_IDENT_STRING="${USER}"
fi

if [[ -z "$ZEPPELIN_INTERPRETER_REMOTE_RUNNER" ]]; then
export ZEPPELIN_INTERPRETER_REMOTE_RUNNER="bin/interpreter.cmd"
fi
11 changes: 8 additions & 3 deletions bin/interpreter.cmd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ if /I "%~1"=="-d" (
if /I "%~1"=="-p" set PORT=%~2
if /I "%~1"=="-c" set CALLBACK_HOST=%~2
if /I "%~1"=="-l" set LOCAL_INTERPRETER_REPO=%~2
if /I "%~1"=="-i" set INTERPRETER_GROUP_ID=%~2
if /I "%~1"=="-r" set INTERPRETER_PORT=%~2
shift
goto loop
:cont
Expand Down Expand Up @@ -59,6 +61,8 @@ set ZEPPELIN_SERVER=org.apache.zeppelin.interpreter.remote.RemoteInterpreterServ

set ZEPPELIN_LOGFILE=%ZEPPELIN_LOG_DIR%\zeppelin-interpreter-%INTERPRETER_ID%-%ZEPPELIN_IDENT_STRING%-%HOSTNAME%.log

set SCALA_VERSION=2.11

if not exist "%ZEPPELIN_LOG_DIR%" (
echo Log dir doesn't exist, create %ZEPPELIN_LOG_DIR%
mkdir "%ZEPPELIN_LOG_DIR%"
Expand All @@ -67,7 +71,7 @@ if not exist "%ZEPPELIN_LOG_DIR%" (
if /I "%INTERPRETER_ID%"=="spark" (
if defined SPARK_HOME (
set SPARK_SUBMIT=%SPARK_HOME%\bin\spark-submit.cmd
for %%d in ("%ZEPPELIN_HOME%\interpreter\spark\zeppelin-spark*.jar") do (
for %%d in ("%ZEPPELIN_HOME%\interpreter\spark\spark-interpreter*.jar") do (
set SPARK_APP_JAR=%%d
)
set ZEPPELIN_CLASSPATH="!SPARK_APP_JAR!"
Expand All @@ -90,6 +94,7 @@ if /I "%INTERPRETER_ID%"=="spark" (
)

call "%bin%\functions.cmd" ADDJARINDIR "%INTERPRETER_DIR%\dep"
call "%bin%\functions.cmd" ADDJARINDIR "%INTERPRETER_DIR%\scala-%SCALA_VERSION%"

for %%d in ("%ZEPPELIN_HOME%\interpreter\spark\pyspark\py4j-*-src.zip") do (
set py4j=%%d
Expand Down Expand Up @@ -128,11 +133,11 @@ if not defined ZEPPELIN_CLASSPATH_OVERRIDES (
if defined SPARK_SUBMIT (
set JAVA_INTP_OPTS=%JAVA_INTP_OPTS% -Dzeppelin.log.file='%ZEPPELIN_LOGFILE%'

"%SPARK_SUBMIT%" --class %ZEPPELIN_SERVER% --jars %CLASSPATH% --driver-java-options "!JAVA_INTP_OPTS!" %SPARK_SUBMIT_OPTIONS% "%SPARK_APP_JAR%" "%CALLBACK_HOST%" %PORT%
"%SPARK_SUBMIT%" --class %ZEPPELIN_SERVER% --jars %CLASSPATH% --driver-java-options "!JAVA_INTP_OPTS!" %SPARK_SUBMIT_OPTIONS% "%SPARK_APP_JAR%" "%CALLBACK_HOST%" %PORT% %INTERPRETER_GROUP_ID% %INTERPRETER_PORT%
) else (
set JAVA_INTP_OPTS=%JAVA_INTP_OPTS% -Dzeppelin.log.file="%ZEPPELIN_LOGFILE%"

"%ZEPPELIN_RUNNER%" !JAVA_INTP_OPTS! %ZEPPELIN_INTP_MEM% -cp '%ZEPPELIN_CLASSPATH_OVERRIDES%;%CLASSPATH%' %ZEPPELIN_SERVER% "%CALLBACK_HOST%" %PORT%
"%ZEPPELIN_RUNNER%" !JAVA_INTP_OPTS! %ZEPPELIN_INTP_MEM% -cp '%ZEPPELIN_CLASSPATH_OVERRIDES%;%CLASSPATH%' %ZEPPELIN_SERVER% "%CALLBACK_HOST%" %PORT% %INTERPRETER_GROUP_ID% %INTERPRETER_PORT%
)

exit /b
Expand Down
Loading