From 2c7f55e5b0e7c09139bc50dddb528a797851e6e6 Mon Sep 17 00:00:00 2001 From: Ilya Muradyan Date: Thu, 7 Nov 2019 13:42:29 +0300 Subject: [PATCH] Some fixes of building and running under Windows --- bin/common-win.sh | 147 ++++++++++ bin/interpreter.cmd | 11 +- bin/zeppelin-daemon-win.sh | 266 ++++++++++++++++++ .../zeppelin/conf/ZeppelinConfiguration.java | 2 +- .../notebook/repo/VFSNotebookRepo.java | 18 +- 5 files changed, 437 insertions(+), 7 deletions(-) create mode 100644 bin/common-win.sh create mode 100644 bin/zeppelin-daemon-win.sh diff --git a/bin/common-win.sh b/bin/common-win.sh new file mode 100644 index 00000000000..5eedacd6bef --- /dev/null +++ b/bin/common-win.sh @@ -0,0 +1,147 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +if [ -L ${BASH_SOURCE-$0} ]; then + FWDIR=$(dirname $(readlink "${BASH_SOURCE-$0}")) +else + FWDIR=$(dirname "${BASH_SOURCE-$0}") +fi + +if [[ -z "${ZEPPELIN_HOME}" ]]; then + # Make ZEPPELIN_HOME look cleaner in logs by getting rid of the + # extra ../ + export ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)" + export ZEPPELIN_HOME="${ZEPPELIN_HOME/\/c\//C:\/}" +fi + +if [[ -z "${ZEPPELIN_CONF_DIR}" ]]; then + export ZEPPELIN_CONF_DIR="${ZEPPELIN_HOME}/conf" +fi + +if [[ -z "${ZEPPELIN_LOG_DIR}" ]]; then + export ZEPPELIN_LOG_DIR="${ZEPPELIN_HOME}/logs" +fi + +if [[ -z "$ZEPPELIN_PID_DIR" ]]; then + export ZEPPELIN_PID_DIR="${ZEPPELIN_HOME}/run" +fi + +if [[ -z "${ZEPPELIN_WAR}" ]]; then + if [[ -d "${ZEPPELIN_HOME}/zeppelin-web/dist" ]]; then + export ZEPPELIN_WAR="${ZEPPELIN_HOME}/zeppelin-web/dist" + else + export ZEPPELIN_WAR=$(find -L "${ZEPPELIN_HOME}" -name "zeppelin-web*.war") + fi +fi + +if [[ -f "${ZEPPELIN_CONF_DIR}/zeppelin-env.sh" ]]; then + . "${ZEPPELIN_CONF_DIR}/zeppelin-env.sh" +fi + +ZEPPELIN_CLASSPATH+=";${ZEPPELIN_CONF_DIR}" + +function addEachJarInDir(){ + if [[ -d "${1}" ]]; then + for jar in $(find -L "${1}" -maxdepth 1 -name '*jar'); do + ZEPPELIN_CLASSPATH="$jar;$ZEPPELIN_CLASSPATH" + done + fi +} + +function addEachJarInDirRecursive(){ + if [[ -d "${1}" ]]; then + for jar in $(find -L "${1}" -type f -name '*jar'); do + ZEPPELIN_CLASSPATH="$jar;$ZEPPELIN_CLASSPATH" + done + fi +} + +function addEachJarInDirRecursiveForIntp(){ + if [[ -d "${1}" ]]; then + for jar in $(find -L "${1}" -type f -name '*jar'); do + ZEPPELIN_INTP_CLASSPATH="$jar;$ZEPPELIN_INTP_CLASSPATH" + done + fi +} + +function addJarInDir(){ + if [[ -d "${1}" ]]; then + ZEPPELIN_CLASSPATH="${1}/*;${ZEPPELIN_CLASSPATH}" + fi +} + +function addJarInDirForIntp() { + if [[ -d "${1}" ]]; then + ZEPPELIN_INTP_CLASSPATH="${1}/*;${ZEPPELIN_INTP_CLASSPATH}" + fi +} + +ZEPPELIN_COMMANDLINE_MAIN=org.apache.zeppelin.utils.CommandLineUtils + +function getZeppelinVersion(){ + if [[ -d "${ZEPPELIN_HOME}/zeppelin-server/target/classes" ]]; then + ZEPPELIN_CLASSPATH+=";${ZEPPELIN_HOME}/zeppelin-server/target/classes" + fi + addJarInDir "${ZEPPELIN_HOME}/zeppelin-server/target/lib" + CLASSPATH+=";${ZEPPELIN_CLASSPATH}" + $ZEPPELIN_RUNNER -cp $CLASSPATH $ZEPPELIN_COMMANDLINE_MAIN -v + exit 0 +} + +# Text encoding for +# read/write job into files, +# receiving/displaying query/result. +if [[ -z "${ZEPPELIN_ENCODING}" ]]; then + export ZEPPELIN_ENCODING="UTF-8" +fi + +if [[ -z "${ZEPPELIN_MEM}" ]]; then + export ZEPPELIN_MEM="-Xms1024m -Xmx1024m -XX:MaxPermSize=512m" +fi + +if [[ -z "${ZEPPELIN_INTP_MEM}" ]]; then + export ZEPPELIN_INTP_MEM="-Xms1024m -Xmx1024m -XX:MaxPermSize=512m" +fi + +JAVA_OPTS+=" ${ZEPPELIN_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING} ${ZEPPELIN_MEM}" +JAVA_OPTS+=" -Dlog4j.configuration=file:///${ZEPPELIN_CONF_DIR}/log4j.properties" +export JAVA_OPTS + +JAVA_INTP_OPTS="${ZEPPELIN_INTP_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING}" +if [[ -z "${ZEPPELIN_SPARK_YARN_CLUSTER}" ]]; then + JAVA_INTP_OPTS+=" -Dlog4j.configuration=file:///${ZEPPELIN_CONF_DIR}/log4j.properties" +else + JAVA_INTP_OPTS+=" -Dlog4j.configuration=log4j_yarn_cluster.properties" +fi +export JAVA_INTP_OPTS + + +if [[ -n "${JAVA_HOME}" ]]; then + ZEPPELIN_RUNNER="${JAVA_HOME}\\bin\\java" +else + ZEPPELIN_RUNNER=java +fi +export ZEPPELIN_RUNNER + +if [[ -z "$ZEPPELIN_IDENT_STRING" ]]; then + export ZEPPELIN_IDENT_STRING="${USER}" +fi + +if [[ -z "$ZEPPELIN_INTERPRETER_REMOTE_RUNNER" ]]; then + export ZEPPELIN_INTERPRETER_REMOTE_RUNNER="bin/interpreter.cmd" +fi diff --git a/bin/interpreter.cmd b/bin/interpreter.cmd index e9d0b7cb4fc..f81fd1674fd 100644 --- a/bin/interpreter.cmd +++ b/bin/interpreter.cmd @@ -29,6 +29,8 @@ if /I "%~1"=="-d" ( if /I "%~1"=="-p" set PORT=%~2 if /I "%~1"=="-c" set CALLBACK_HOST=%~2 if /I "%~1"=="-l" set LOCAL_INTERPRETER_REPO=%~2 +if /I "%~1"=="-i" set INTERPRETER_GROUP_ID=%~2 +if /I "%~1"=="-r" set INTERPRETER_PORT=%~2 shift goto loop :cont @@ -59,6 +61,8 @@ set ZEPPELIN_SERVER=org.apache.zeppelin.interpreter.remote.RemoteInterpreterServ set ZEPPELIN_LOGFILE=%ZEPPELIN_LOG_DIR%\zeppelin-interpreter-%INTERPRETER_ID%-%ZEPPELIN_IDENT_STRING%-%HOSTNAME%.log +set SCALA_VERSION=2.11 + if not exist "%ZEPPELIN_LOG_DIR%" ( echo Log dir doesn't exist, create %ZEPPELIN_LOG_DIR% mkdir "%ZEPPELIN_LOG_DIR%" @@ -67,7 +71,7 @@ if not exist "%ZEPPELIN_LOG_DIR%" ( if /I "%INTERPRETER_ID%"=="spark" ( if defined SPARK_HOME ( set SPARK_SUBMIT=%SPARK_HOME%\bin\spark-submit.cmd - for %%d in ("%ZEPPELIN_HOME%\interpreter\spark\zeppelin-spark*.jar") do ( + for %%d in ("%ZEPPELIN_HOME%\interpreter\spark\spark-interpreter*.jar") do ( set SPARK_APP_JAR=%%d ) set ZEPPELIN_CLASSPATH="!SPARK_APP_JAR!" @@ -90,6 +94,7 @@ if /I "%INTERPRETER_ID%"=="spark" ( ) call "%bin%\functions.cmd" ADDJARINDIR "%INTERPRETER_DIR%\dep" + call "%bin%\functions.cmd" ADDJARINDIR "%INTERPRETER_DIR%\scala-%SCALA_VERSION%" for %%d in ("%ZEPPELIN_HOME%\interpreter\spark\pyspark\py4j-*-src.zip") do ( set py4j=%%d @@ -128,11 +133,11 @@ if not defined ZEPPELIN_CLASSPATH_OVERRIDES ( if defined SPARK_SUBMIT ( set JAVA_INTP_OPTS=%JAVA_INTP_OPTS% -Dzeppelin.log.file='%ZEPPELIN_LOGFILE%' - "%SPARK_SUBMIT%" --class %ZEPPELIN_SERVER% --jars %CLASSPATH% --driver-java-options "!JAVA_INTP_OPTS!" %SPARK_SUBMIT_OPTIONS% "%SPARK_APP_JAR%" "%CALLBACK_HOST%" %PORT% + "%SPARK_SUBMIT%" --class %ZEPPELIN_SERVER% --jars %CLASSPATH% --driver-java-options "!JAVA_INTP_OPTS!" %SPARK_SUBMIT_OPTIONS% "%SPARK_APP_JAR%" "%CALLBACK_HOST%" %PORT% %INTERPRETER_GROUP_ID% %INTERPRETER_PORT% ) else ( set JAVA_INTP_OPTS=%JAVA_INTP_OPTS% -Dzeppelin.log.file="%ZEPPELIN_LOGFILE%" - "%ZEPPELIN_RUNNER%" !JAVA_INTP_OPTS! %ZEPPELIN_INTP_MEM% -cp '%ZEPPELIN_CLASSPATH_OVERRIDES%;%CLASSPATH%' %ZEPPELIN_SERVER% "%CALLBACK_HOST%" %PORT% + "%ZEPPELIN_RUNNER%" !JAVA_INTP_OPTS! %ZEPPELIN_INTP_MEM% -cp '%ZEPPELIN_CLASSPATH_OVERRIDES%;%CLASSPATH%' %ZEPPELIN_SERVER% "%CALLBACK_HOST%" %PORT% %INTERPRETER_GROUP_ID% %INTERPRETER_PORT% ) exit /b diff --git a/bin/zeppelin-daemon-win.sh b/bin/zeppelin-daemon-win.sh new file mode 100644 index 00000000000..4613aece4b3 --- /dev/null +++ b/bin/zeppelin-daemon-win.sh @@ -0,0 +1,266 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# description: Start and stop daemon script for. +# + +USAGE="-e Usage: zeppelin-daemon.sh\n\t + [--config ] {start|stop|upstart|restart|reload|status}\n\t + [--version | -v]" + +if [[ "$1" == "--config" ]]; then + shift + conf_dir="$1" + if [[ ! -d "${conf_dir}" ]]; then + echo "ERROR : ${conf_dir} is not a directory" + echo ${USAGE} + exit 1 + else + export ZEPPELIN_CONF_DIR="${conf_dir}" + fi + shift +fi + +if [ -L ${BASH_SOURCE-$0} ]; then + BIN=$(dirname $(readlink "${BASH_SOURCE-$0}")) +else + BIN=$(dirname ${BASH_SOURCE-$0}) +fi +BIN=$(cd "${BIN}">/dev/null; pwd) + +. "${BIN}/common-win.sh" +. "${BIN}/functions.sh" + +HOSTNAME=$(hostname) +ZEPPELIN_NAME="Zeppelin" +ZEPPELIN_LOGFILE="${ZEPPELIN_LOG_DIR}/zeppelin-${ZEPPELIN_IDENT_STRING}-${HOSTNAME}.log" +ZEPPELIN_OUTFILE="${ZEPPELIN_LOG_DIR}/zeppelin-${ZEPPELIN_IDENT_STRING}-${HOSTNAME}.out" +ZEPPELIN_PID="${ZEPPELIN_PID_DIR}/zeppelin-${ZEPPELIN_IDENT_STRING}-${HOSTNAME}.pid" +ZEPPELIN_MAIN=org.apache.zeppelin.server.ZeppelinServer +JAVA_OPTS+=" -Dzeppelin.log.file=${ZEPPELIN_LOGFILE}" + +# construct classpath +if [[ -d "${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes" ]]; then + ZEPPELIN_CLASSPATH+=";${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes" +fi + +if [[ -d "${ZEPPELIN_HOME}/zeppelin-zengine/target/classes" ]]; then + ZEPPELIN_CLASSPATH+=";${ZEPPELIN_HOME}/zeppelin-zengine/target/classes" +fi + +if [[ -d "${ZEPPELIN_HOME}/zeppelin-server/target/classes" ]]; then + ZEPPELIN_CLASSPATH+=";${ZEPPELIN_HOME}/zeppelin-server/target/classes" +fi + +if [[ -n "${HADOOP_CONF_DIR}" ]] && [[ -d "${HADOOP_CONF_DIR}" ]]; then + ZEPPELIN_CLASSPATH+=";${HADOOP_CONF_DIR}" +fi + +# Add jdbc connector jar +# ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/jdbc/jars/jdbc-connector-jar" + +addJarInDir "${ZEPPELIN_HOME}" +addJarInDir "${ZEPPELIN_HOME}/lib" +addJarInDir "${ZEPPELIN_HOME}/lib/interpreter" +addJarInDir "${ZEPPELIN_HOME}/zeppelin-interpreter/target/lib" +addJarInDir "${ZEPPELIN_HOME}/zeppelin-zengine/target/lib" +addJarInDir "${ZEPPELIN_HOME}/zeppelin-server/target/lib" +addJarInDir "${ZEPPELIN_HOME}/zeppelin-web/target/lib" + +CLASSPATH+=";${ZEPPELIN_CLASSPATH}" + +if [[ "${ZEPPELIN_NICENESS}" = "" ]]; then + export ZEPPELIN_NICENESS=0 +fi + +function initialize_default_directories() { + if [[ ! -d "${ZEPPELIN_LOG_DIR}" ]]; then + echo "Log dir doesn't exist, create ${ZEPPELIN_LOG_DIR}" + $(mkdir -p "${ZEPPELIN_LOG_DIR}") + fi + + if [[ ! -d "${ZEPPELIN_PID_DIR}" ]]; then + echo "Pid dir doesn't exist, create ${ZEPPELIN_PID_DIR}" + $(mkdir -p "${ZEPPELIN_PID_DIR}") + fi +} + +function wait_for_zeppelin_to_die() { + local pid + local count + pid=$1 + timeout=$2 + count=0 + timeoutTime=$(date "+%s") + let "timeoutTime+=$timeout" + currentTime=$(date "+%s") + forceKill=1 + + while [[ $currentTime -lt $timeoutTime ]]; do + $(kill ${pid} > /dev/null 2> /dev/null) + if kill -0 ${pid} > /dev/null 2>&1; then + sleep 3 + else + forceKill=0 + break + fi + currentTime=$(date "+%s") + done + + if [[ forceKill -ne 0 ]]; then + $(kill -9 ${pid} > /dev/null 2> /dev/null) + fi +} + +function wait_zeppelin_is_up_for_ci() { + if [[ "${CI}" == "true" ]]; then + local count=0; + while [[ "${count}" -lt 30 ]]; do + curl -v localhost:8080 2>&1 | grep '200 OK' + if [[ $? -ne 0 ]]; then + sleep 1 + continue + else + break + fi + let "count+=1" + done + fi +} + +function print_log_for_ci() { + if [[ "${CI}" == "true" ]]; then + tail -1000 "${ZEPPELIN_LOGFILE}" | sed 's/^/ /' + fi +} + +function check_if_process_is_alive() { + local pid + pid=$(cat ${ZEPPELIN_PID}) + if ! kill -0 ${pid} >/dev/null 2>&1; then + action_msg "${ZEPPELIN_NAME} process died" "${SET_ERROR}" + print_log_for_ci + return 1 + fi +} + +function upstart() { + + # upstart() allows zeppelin to be run and managed as a service + # for example, this could be called from an upstart script in /etc/init + # where the service manager starts and stops the process + initialize_default_directories + + echo "ZEPPELIN_CLASSPATH: ${ZEPPELIN_CLASSPATH_OVERRIDES}:${CLASSPATH}" >> "${ZEPPELIN_OUTFILE}" + + $ZEPPELIN_RUNNER $JAVA_OPTS -cp $ZEPPELIN_CLASSPATH_OVERRIDES:$CLASSPATH $ZEPPELIN_MAIN >> "${ZEPPELIN_OUTFILE}" +} + +function start() { + local pid + + if [[ -f "${ZEPPELIN_PID}" ]]; then + pid=$(cat ${ZEPPELIN_PID}) + if kill -0 ${pid} >/dev/null 2>&1; then + echo "${ZEPPELIN_NAME} is already running" + return 0; + fi + fi + + initialize_default_directories + + echo "ZEPPELIN_CLASSPATH: ${ZEPPELIN_CLASSPATH_OVERRIDES}:${CLASSPATH}" >> "${ZEPPELIN_OUTFILE}" + + nohup nice -n $ZEPPELIN_NICENESS "$ZEPPELIN_RUNNER" $JAVA_OPTS -classpath "$CLASSPATH" $ZEPPELIN_MAIN >> "${ZEPPELIN_OUTFILE}" 2>&1 < /dev/null & + pid=$! + if [[ -z "${pid}" ]]; then + action_msg "${ZEPPELIN_NAME} start" "${SET_ERROR}" + return 1; + else + action_msg "${ZEPPELIN_NAME} start" "${SET_OK}" + echo ${pid} > ${ZEPPELIN_PID} + fi + + wait_zeppelin_is_up_for_ci + sleep 2 + check_if_process_is_alive +} + +function stop() { + local pid + + # zeppelin daemon kill + if [[ ! -f "${ZEPPELIN_PID}" ]]; then + echo "${ZEPPELIN_NAME} is not running" + else + pid=$(cat ${ZEPPELIN_PID}) + if [[ -z "${pid}" ]]; then + echo "${ZEPPELIN_NAME} is not running" + else + wait_for_zeppelin_to_die $pid 40 + $(rm -f ${ZEPPELIN_PID}) + action_msg "${ZEPPELIN_NAME} stop" "${SET_OK}" + fi + fi +} + +function find_zeppelin_process() { + local pid + + if [[ -f "${ZEPPELIN_PID}" ]]; then + pid=$(cat ${ZEPPELIN_PID}) + if ! kill -0 ${pid} > /dev/null 2>&1; then + action_msg "${ZEPPELIN_NAME} running but process is dead" "${SET_ERROR}" + return 1 + else + action_msg "${ZEPPELIN_NAME} is running" "${SET_OK}" + fi + else + action_msg "${ZEPPELIN_NAME} is not running" "${SET_ERROR}" + return 1 + fi +} + +case "${1}" in + start) + start + ;; + stop) + stop + ;; + upstart) + upstart + ;; + reload) + stop + start + ;; + restart) + echo "${ZEPPELIN_NAME} is restarting" >> "${ZEPPELIN_OUTFILE}" + stop + start + ;; + status) + find_zeppelin_process + ;; + -v | --version) + getZeppelinVersion + ;; + *) + echo ${USAGE} +esac diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index db6f09d45bd..edd590811f1 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -562,7 +562,7 @@ public String getInterpreterPortRange() { } public boolean isWindowsPath(String path){ - return path.matches("^[A-Za-z]:\\\\.*"); + return path.matches("^[A-Za-z]:[\\\\/].*"); } public boolean isAnonymousAllowed() { diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java index d8d8df0433d..0fe4b69a99d 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java @@ -29,8 +29,8 @@ import java.util.Map; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; +import org.apache.commons.vfs2.FileName; import org.apache.commons.vfs2.FileObject; -import org.apache.commons.vfs2.FileSystemException; import org.apache.commons.vfs2.FileSystemManager; import org.apache.commons.vfs2.NameScope; import org.apache.commons.vfs2.Selectors; @@ -88,7 +88,19 @@ protected void setNotebookDirectory(String notebookDirPath) throws IOException { LOGGER.info("Notebook dir doesn't exist: {}, creating it.", rootNotebookFileObject.getName().getPath()); } - this.rootNotebookFolder = rootNotebookFileObject.getName().getPath(); + this.rootNotebookFolder = getFilePath(rootNotebookFileObject); + } + + static private String getFilePath(final FileObject fileObject) { + FileName fileName = fileObject.getName(); + String rootURI = fileName.getRootURI(); + String root; + if (rootURI.endsWith(":/")) { // Windows + root = rootURI.substring(8, 10); + } else { // *nix + root = ""; + } + return root + fileName.getPath(); } @Override @@ -110,7 +122,7 @@ private Map listFolder(FileObject fileObject) throws IOExcepti noteInfos.putAll(listFolder(child)); } } else { - String noteFileName = fileObject.getName().getPath(); + String noteFileName = getFilePath(fileObject); if (noteFileName.endsWith(".zpln")) { try { String noteId = getNoteId(noteFileName);