khud · dkaznacheev · Jul 4, 2019 · Jul 4, 2019 · Jul 4, 2019 · Jul 8, 2019
diff --git a/.travis.yml b/.travis.yml
@@ -106,13 +106,13 @@ matrix:
     - sudo: required
       jdk: "openjdk8"
       dist: xenial
-      env: BUILD_PLUGINS="true" PYTHON="3" SCALA_VER="2.11" PROFILE="-Pspark-2.4 -Pspark-scala-2.11 -Phadoop2 -Pintegration" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl zeppelin-interpreter-integration,jdbc,zeppelin-web,spark/spark-dependencies,markdown" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest24,SparkIntegrationTest24,JdbcIntegrationTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
+      env: BUILD_PLUGINS="true" PYTHON="3" SCALA_VER="2.11" PROFILE="-Pspark-2.4 -Pspark-scala-2.11 -Phadoop2 -Pintegration" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl zeppelin-interpreter-integration,jdbc,zeppelin-web,spark/spark-dependencies,markdown" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest24,SparkIntegrationTest24,JdbcIntegrationTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.kotlin.* -DfailIfNoTests=false"
 
     # ZeppelinSparkClusterTest24, SparkIntegrationTest24, JdbcIntegrationTest, Unit test of Spark 2.4 (Scala-2.12)
     - sudo: required
       jdk: "openjdk8"
       dist: xenial
-      env: BUILD_PLUGINS="true" PYTHON="3" SCALA_VER="2.12" PROFILE="-Pspark-2.4 -Pspark-scala-2.12 -Phadoop2 -Pintegration" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl zeppelin-interpreter-integration,jdbc,zeppelin-web,spark/spark-dependencies,markdown" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest24,SparkIntegrationTest24,JdbcIntegrationTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
+      env: BUILD_PLUGINS="true" PYTHON="3" SCALA_VER="2.12" PROFILE="-Pspark-2.4 -Pspark-scala-2.12 -Phadoop2 -Pintegration" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl zeppelin-interpreter-integration,jdbc,zeppelin-web,spark/spark-dependencies,markdown" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest24,SparkIntegrationTest24,JdbcIntegrationTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.kotlin.* -DfailIfNoTests=false"
 
     # ZeppelinSparkClusterTest23, SparkIntegrationTest23, Unit test of Spark 2.3 (Scala-2.11) and Unit test PythonInterpreter under python2
     - sudo: required

diff --git a/LICENSE b/LICENSE
@@ -262,6 +262,7 @@ The text of each license is also included at licenses/LICENSE-[project]-[version
     (Apache 2.0) Google Cloud Client Library for Java (https://github.com/GoogleCloudPlatform/google-cloud-java)
     (Apache 2.0) concurrentunit (https://github.com/jhalterman/concurrentunit)
     (Apache 2.0) Embedded MongoDB (https://github.com/flapdoodle-oss/de.flapdoodle.embed.mongo)
+    (Apache 2.0) Kotlin (https://github.com/JetBrains/kotlin)
 
 ========================================================================
 BSD 3-Clause licenses

diff --git a/bin/common.sh b/bin/common.sh
@@ -114,7 +114,7 @@ if [[ -z "${ZEPPELIN_MEM}" ]]; then
 fi
 
 if [[ -z "${ZEPPELIN_INTP_MEM}" ]]; then
-  export ZEPPELIN_INTP_MEM="-Xms1024m -Xmx1024m -XX:MaxPermSize=512m"
+  export ZEPPELIN_INTP_MEM="-Xms1024m -Xmx2048m -XX:MaxPermSize=512m"
 fi
 
 JAVA_OPTS+=" ${ZEPPELIN_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING} ${ZEPPELIN_MEM}"

diff --git a/conf/interpreter-list b/conf/interpreter-list
@@ -32,6 +32,7 @@ hbase           org.apache.zeppelin:zeppelin-hbase:0.9.0                Hbase in
 ignite          org.apache.zeppelin:zeppelin-ignite:0.9.0               Ignite interpreter
 java            org.apache.zeppelin:zeppelin-java:0.9.0                 Java interpreter
 jdbc            org.apache.zeppelin:zeppelin-jdbc:0.9.0                 Jdbc interpreter
+kotlin          org.apache.zeppelin:zeppelin-kotlin:0.9.0               Kotlin interpreter
 kylin           org.apache.zeppelin:zeppelin-kylin:0.9.0                Kylin interpreter
 lens            org.apache.zeppelin:zeppelin-lens:0.9.0                 Lens interpreter
 livy            org.apache.zeppelin:zeppelin-livy:0.9.0                 Livy interpreter

diff --git a/docs/_includes/themes/zeppelin/_navigation.html b/docs/_includes/themes/zeppelin/_navigation.html
@@ -142,6 +142,7 @@
                 <li><a href="{{BASE_PATH}}/interpreter/hive.html">Hive</a></li>
                 <li><a href="{{BASE_PATH}}/interpreter/ignite.html">Ignite</a></li>
                 <li><a href="{{BASE_PATH}}/interpreter/java.html">Java</a></li>
+                <li><a href="{{BASE_PATH}}/interpreter/kotlin.html">Kotlin</a></li>
                 <li><a href="{{BASE_PATH}}/interpreter/kylin.html">Kylin</a></li>
                 <li><a href="{{BASE_PATH}}/interpreter/lens.html">Lens</a></li>
                 <li><a href="{{BASE_PATH}}/interpreter/livy.html">Livy</a></li>

diff --git a/docs/index.md b/docs/index.md
@@ -144,6 +144,7 @@ limitations under the License.
   * [Ignite](./interpreter/ignite.html)
   * [JDBC](./interpreter/jdbc.html)
   * [Kylin](./interpreter/kylin.html)
+  * [Kotlin](./interpreter/kotlin.html)
   * [Lens](./interpreter/lens.html)
   * [Livy](./interpreter/livy.html)
   * [Mahout](./interpreter/mahout.html)

diff --git a/docs/interpreter/kotlin.md b/docs/interpreter/kotlin.md
@@ -0,0 +1,79 @@
+---
+layout: page
+title: "Kotlin interpreter in Apache Zeppelin"
+description: "Kotlin is a cross-platform, statically typed, general-purpose programming language with type inference."
+group: interpreter
+---
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+{% include JB/setup %}
+
+# Kotlin interpreter for Apache Zeppelin
+
+<div id="toc"></div>
+
+## Overview
+Kotlin is a cross-platform, statically typed, general-purpose programming language with type inference.
+It is designed to interoperate fully with Java, and the JVM version of its standard library depends on the Java Class Library, but type inference allows its syntax to be more concise.
+
+## Configuration
+<table class="table-configuration">
+  <tr>
+    <th>Name</th>
+    <th>Default</th>
+    <th>Description</th>
+  </tr>
+  <tr>
+    <td>zeppelin.kotlin.maxResult</td>
+    <td>1000</td>
+    <td>Max n
+  <tr>
+    <td>zeppelin.kotlin.shortenTypes</td>
+    <td>true</td>
+    <td>Display shortened types instead of full, e.g. Int vs kotlin.Int</td>
+  </tr>
+</table>
+
+## Example
+
+```kotlin
+%kotlin 
+
+fun square(n: Int): Int = n * n
+```
+
+## Kotlin Context
+Kotlin context is accessible via `kc` object bound to the interpreter. 
+It holds `vars` and `functions` fields that return all user-defined variables and functions present in the interpreter.
+You can also print variables or functions by calling `kc.showVars()` or `kc.showFunctions()`.
+
+### Example
+
+
+```kotlin
+fun square(n: Int): Int = n * n
+
+val greeter = { s: String -> println("Hello $s!") }
+val l = listOf("Drive", "to", "develop")
+
+kc.showVars()
+kc.showFunctions()
+```
+Output:
+```
+l: List<String> = [Drive, to, develop]
+greeter: (String) -> Unit = (kotlin.String) -> kotlin.Unit
+fun square(Int): Int
+```
diff --git a/docs/interpreter/spark.md b/docs/interpreter/spark.md
@@ -26,7 +26,7 @@ limitations under the License.
 ## Overview
 [Apache Spark](http://spark.apache.org) is a fast and general-purpose cluster computing system.
 It provides high-level APIs in Java, Scala, Python and R, and an optimized engine that supports general execution graphs.
-Apache Spark is supported in Zeppelin with Spark interpreter group which consists of below five interpreters.
+Apache Spark is supported in Zeppelin with Spark interpreter group which consists of below six interpreters.
 
 <table class="table-configuration">
   <tr>
@@ -39,6 +39,11 @@ Apache Spark is supported in Zeppelin with Spark interpreter group which consist
     <td>SparkInterpreter</td>
     <td>Creates a SparkContext and provides a Scala environment</td>
   </tr>
+  <tr>
+    <td>%spark.kotlin</td>
+    <td>KotlinSparkInterpreter</td>
+    <td>Provides a Kotlin environment</td>
+  </tr>
   <tr>
     <td>%spark.pyspark</td>
     <td>PySparkInterpreter</td>
@@ -217,7 +222,7 @@ in interpreter setting page means you can use multiple versions of `spark` & `ha
 Starting from 0.9, we totally removed the old spark interpreter implementation, and make the new spark interpreter as the official spark interpreter.
 
 ## SparkContext, SQLContext, SparkSession, ZeppelinContext
-SparkContext, SQLContext and ZeppelinContext are automatically created and exposed as variable names `sc`, `sqlContext` and `z`, respectively, in Scala, Python and R environments.
+SparkContext, SQLContext and ZeppelinContext are automatically created and exposed as variable names `sc`, `sqlContext` and `z`, respectively, in Scala, Kotlin, Python and R environments.
 Staring from 0.6.1 SparkSession is available as variable `spark` when you are using Spark 2.x.
 
 > Note that Scala/Python/R environment shares the same SparkContext, SQLContext and ZeppelinContext instance.

diff --git a/docs/usage/interpreter/installation.md b/docs/usage/interpreter/installation.md
@@ -188,6 +188,11 @@ You can also find the below community managed interpreter list in `conf/interpre
     <td>org.apache.zeppelin:zeppelin-jdbc:0.9.0</td>
     <td>Jdbc interpreter</td>
   </tr>
+  <tr>
+    <td>kotlin</td>
+    <td>org.apache.zeppelin:zeppelin-kotlin:0.7.0</td>
+    <td>Kotlin interpreter</td>
+  </tr>
   <tr>
     <td>kylin</td>
     <td>org.apache.zeppelin:zeppelin-kylin:0.9.0</td>

diff --git a/kotlin/README.md b/kotlin/README.md
@@ -0,0 +1,83 @@
+# Developer guide to Kotlin interpreter
+
+The following module adds Kotlin language support to Apache Zeppelin.
+Here is the guide to its implementation and how it can be improved and tested. 
+
+## Implementation details
+### Kotlin REPL
+For interactive Kotlin execution, an instance of `KotlinRepl` is created.
+To set REPL properties (such as classpath, generated classes output directory, max result, etc.),
+pass `KotlinReplProperties` to its constructor. For example:
+```$java
+KotlinReplProperties replProperties = new KotlinReplProperties()
+    .maxResult(1000)
+    .shortenTypes(true);
+KotlinRepl repl = new KotlinRepl(replProperties);
+```
+
+### Variable/function binding
+You can also bind variables and functions on REPL creation using implicit receiver language feature.
+This means that all code run in REPL will be executed in Kotlin's `with` block with the receiver, 
+making the receiver's fields and methods accessible.
+
+To add your variables/functions, extend `KotlinReceiver` class (in separate file), declare your fields and methods, and pass an instance of it to
+`KotlinReplProperties`. Example:
+```$java
+// In separate file:
+class CustomReceiver extends KotlinReceiver {
+    public int myValue = 1 // will be converted to Kotlin "var myValue: Int"
+    public final String messageTemplate = "Value = %VALUE%" // "val messageTemplate: String"
+
+    public String getMessage() {
+        return messageTemplate.replace("%VALUE%", String.valueOf(myValue));
+    }
+}
+
+// In intepreter creation:
+replProperties.receiver(new CustomReceiver);
+KotlinRepl repl = new KotlinRepl(replProperties);
+repl.eval("getMessage()"); // will return interpreterResult with "Value = 1" string
+``` 
+
+In `KotlinInterpreter` REPL properties are created on construction, are accessible via `getKotlinReplProperties` method,
+and are used in REPL creation on `open()`.
+
+### Generated class files
+Each code snippet run in REPL is registered as a separate class and saved in location 
+specified by `outputDir` REPL property. Anonymous classes and lambdas also get saved there under specific names. 
+
+This is needed for Spark to send classes to remote executors, and in Spark Kotlin interpreter this directory is the same 
+as in `sparkContext` option `spark.repl.class.outputDir`.
+
+### Kotlin Spark Interpreter
+Kotlin interpreter in Spark intepreter group takes `SparkSession`, `JavaSparkContext`, `SQLContext` 
+and `BaseZeppelinContext` from `SparkInterpreter` in the same session and binds them in its scope.
+
+## Testing
+Kotlin Interpreter and Spark Kotlin Interpreter come with unit tests. 
+They can be run with \
+`mvn clean test` \
+in `$ZEPPELIN_HOME/kotlin` for base Kotlin Interpreter and \
+`mvn -Dtest=KotlinSparkInterpreterTest test` \
+in `$ZEPPELIN_HOME/spark/interpreter` for Spark Kotlin Interpreter.
+
+To test manually, build Zeppelin with \
+`mvn clean package -DskipTests` \
+and create a note with `kotlin` interpreter for base or `spark` for Spark. 
+In Spark interpreter, add `%spark.kotlin` in the start of paragraph to use Kotlin Spark Interpreter.
+
+Example:
+```$kotlin
+%spark.kotlin
+val df = spark.range(10)
+df.show()
+```
+## Possible Improvements
+* It would be great to bind `ZeppelinContext` to base Kotlin interpreter, but for now I had trouble instantiating it 
+inside KotlinInterpreter.
+* When Kotlin has its own Spark API, it will be good to move to it. Currently in Java Spark API Kotlin 
+can not use things like `forEach` because of ambiguity between `Iterable<?>.forEach` and `Map<?, ?>.forEach` 
+(`foreach` from Spark's API does work, though).
+* The scoped mode for Kotlin Spark Interpreter currently has issues with having the same class output directory 
+for different intepreters, leading to overwriting classes. Adding prefixes to generated classes or putting them
+ in separate directories leads to `ClassNotFoundException` on Spark executors.
diff --git a/kotlin/pom.xml b/kotlin/pom.xml
@@ -0,0 +1,125 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements. See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License. You may obtain a copy of the License at
+  ~
+  ~ http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <artifactId>zeppelin-interpreter-parent</artifactId>
+        <groupId>org.apache.zeppelin</groupId>
+        <version>0.9.0-SNAPSHOT</version>
+        <relativePath>../zeppelin-interpreter-parent/pom.xml</relativePath>
+    </parent>
+
+    <groupId>org.apache.zeppelin</groupId>
+    <artifactId>zeppelin-kotlin</artifactId>
+    <packaging>jar</packaging>
+    <version>0.9.0-SNAPSHOT</version>
+    <name>Zeppelin: Kotlin interpreter</name>
+
+    <properties>
+        <interpreter.name>kotlin</interpreter.name>
+        <kotlin.version>1.3.50</kotlin.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.jetbrains.kotlin</groupId>
+            <artifactId>kotlin-scripting-jvm-host-embeddable</artifactId>
+            <version>${kotlin.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.jetbrains.kotlin</groupId>
+                    <artifactId>kotlin-stdlib</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.jetbrains.kotlin</groupId>
+                    <artifactId>kotlin-stdlib-common</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.jetbrains.kotlin</groupId>
+            <artifactId>kotlin-compiler-embeddable</artifactId>
+            <version>${kotlin.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.jetbrains.kotlin</groupId>
+                    <artifactId>kotlin-stdlib</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.jetbrains.kotlin</groupId>
+            <artifactId>kotlin-scripting-jvm</artifactId>
+            <version>${kotlin.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.jetbrains.kotlin</groupId>
+                    <artifactId>kotlin-stdlib</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.jetbrains.kotlin</groupId>
+            <artifactId>kotlin-scripting-compiler-embeddable</artifactId>
+            <version>${kotlin.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.jetbrains.kotlin</groupId>
+                    <artifactId>kotlin-stdlib</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.jetbrains.kotlin</groupId>
+            <artifactId>kotlin-stdlib</artifactId>
+            <version>${kotlin.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.jetbrains.kotlin</groupId>
+            <artifactId>kotlin-reflect</artifactId>
+            <version>${kotlin.version}</version>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <artifactId>maven-enforcer-plugin</artifactId>
+            </plugin>
+            <plugin>
+                <artifactId>maven-dependency-plugin</artifactId>
+            </plugin>
+            <plugin>
+                <artifactId>maven-resources-plugin</artifactId>
+            </plugin>
+            <plugin>
+                <artifactId>maven-shade-plugin</artifactId>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-checkstyle-plugin</artifactId>
+                <configuration>
+                    <skip>false</skip>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+</project>