Merge develop into cascading3

.gitignore

@@ -13,6 +13,12 @@ project/plugins/lib_managed/
 project/plugins/src_managed/
 /.idea/
 /.idea_modules/
+.project
+.classpath
+.cache-main
+.cache-tests
+.tmpBin
+bin
 *.iml
 sonatype.sbt
 tutorial/data/cofollows.tsv

.travis.yml

@@ -27,63 +27,63 @@ addons:
 matrix:
   include:
 #BASE TESTS
-    - scala: 2.11.8
-      env: BUILD="base" TEST_TARGET="scalding-args scalding-date maple"
+    - scala: 2.11.11
+      env: BUILD="base" TEST_TARGET="scalding-args scalding-date maple scalding-quotation"
       script: "scripts/run_test.sh"
 
-    - scala: 2.12.1
-      env: BUILD="base" TEST_TARGET="scalding-args scalding-date maple"
+    - scala: 2.12.3
+      env: BUILD="base" TEST_TARGET="scalding-args scalding-date maple scalding-quotation"
       script: "scripts/run_test.sh"
 
-    - scala: 2.11.8
+    - scala: 2.11.11
       env: BUILD="base" TEST_TARGET="scalding-avro scalding-hraven scalding-commons scalding-parquet scalding-parquet-cascading scalding-parquet-scrooge scalding-parquet-scrooge-cascading"
       script: "scripts/run_test.sh"
 
-    - scala: 2.12.1
+    - scala: 2.12.3
       env: BUILD="base" TEST_TARGET="scalding-avro scalding-hraven scalding-commons scalding-parquet scalding-parquet-cascading scalding-parquet-scrooge scalding-parquet-scrooge-cascading"
       script: "scripts/run_test.sh"
 
-    - scala: 2.11.8
+    - scala: 2.11.11
       env: BUILD="base" TEST_TARGET="scalding-core scalding-jdbc scalding-json scalding-db"
       script: "scripts/run_test.sh"
 
-    - scala: 2.12.1
+    - scala: 2.12.3
       env: BUILD="base" TEST_TARGET="scalding-core scalding-jdbc scalding-json scalding-db"
       script: "scripts/run_test.sh"
 
-    - scala: 2.11.8
+    - scala: 2.11.11
       env: BUILD="base" TEST_TARGET="scalding-hadoop-test"
       script: "scripts/run_test.sh"
 
-    - scala: 2.12.1
+    - scala: 2.12.3
       env: BUILD="base" TEST_TARGET="scalding-hadoop-test"
       script: "scripts/run_test.sh"
 
-    - scala: 2.11.8
+    - scala: 2.11.11
       env: BUILD="base" TEST_TARGET="scalding-estimators-test"
       script: "scripts/run_test.sh"
 
-    - scala: 2.12.1
+    - scala: 2.12.3
       env: BUILD="base" TEST_TARGET="scalding-estimators-test"
       script: "scripts/run_test.sh"
 
-    - scala: 2.11.8
+    - scala: 2.11.11
       env: BUILD="base" TEST_TARGET="scalding-serialization"
       script: "scripts/run_test.sh"
 
-    - scala: 2.12.1
+    - scala: 2.12.3
       env: BUILD="base" TEST_TARGET="scalding-serialization"
       script: "scripts/run_test.sh"
 
-    - scala: 2.11.8
+    - scala: 2.11.11
       env: BUILD="base" TEST_TARGET="scalding-thrift-macros"
       script: "scripts/run_test.sh"
 
-    - scala: 2.12.1
+    - scala: 2.12.3
       env: BUILD="base" TEST_TARGET="scalding-thrift-macros"
       script: "scripts/run_test.sh"
 
-    - scala: 2.11.8
+    - scala: 2.11.11
       env: BUILD="test tutorials and matrix tutorials and repl" TEST_TARGET="scalding-repl"
       script:
       - "scripts/run_test.sh"
@@ -92,7 +92,7 @@ matrix:
       - "scripts/build_assembly_no_test.sh scalding-assembly"
       - "scripts/test_matrix_tutorials.sh"
 
-    - scala: 2.12.1
+    - scala: 2.12.3
       env: BUILD="test tutorials and matrix tutorials and repl" TEST_TARGET="scalding-repl"
       script:
       - "scripts/run_test.sh"
@@ -101,7 +101,7 @@ matrix:
       - "scripts/build_assembly_no_test.sh scalding-assembly"
       - "scripts/test_matrix_tutorials.sh"
 
-    - scala: 2.11.8
+    - scala: 2.11.11
       env: BUILD="test repl and typed tutorials and microsite"
       script:
       - ./sbt ++$TRAVIS_SCALA_VERSION clean docs/makeMicrosite
@@ -112,7 +112,7 @@ matrix:
       - "scripts/build_assembly_no_test.sh execution-tutorial"
       - "scripts/test_execution_tutorial.sh"
 
-    - scala: 2.12.1
+    - scala: 2.12.3
       env: BUILD="test repl and typed tutorials"
       script:
       - "scripts/build_assembly_no_test.sh scalding-repl"

build.sbt

@@ -21,6 +21,7 @@ val avroVersion = "1.7.4"
 val bijectionVersion = "0.9.5"
 val cascadingAvroVersion = "2.1.2"
 val chillVersion = "0.8.4"
+val dagonVersion = "0.2.2"
 val elephantbirdVersion = "4.15"
 val hadoopLzoVersion = "0.4.19"
 val hadoopVersion = "2.6.0"
@@ -47,17 +48,19 @@ val printDependencyClasspath = taskKey[Unit]("Prints location of the dependencie
 val sharedSettings = assemblySettings ++ scalariformSettings ++ Seq(
   organization := "com.twitter",
 
-  scalaVersion := "2.11.8",
+  scalaVersion := "2.11.11",
 
-  crossScalaVersions := Seq(scalaVersion.value, "2.12.1"),
+  crossScalaVersions := Seq(scalaVersion.value, "2.12.3"),
 
   ScalariformKeys.preferences := formattingPreferences,
 
   javacOptions ++= Seq("-source", "1.6", "-target", "1.6"),
 
   javacOptions in doc := Seq("-source", "1.6"),
 
-  wartremoverErrors in (Compile, compile) += Wart.OptionPartial,
+  wartremoverErrors in (Compile, compile) ++= Seq(
+    Wart.OptionPartial, Wart.ExplicitImplicitTypes, Wart.LeakingSealed,
+    Wart.Return, Wart.EitherProjectionPartial),
 
   libraryDependencies ++= Seq(
     "org.mockito" % "mockito-all" % "1.8.5" % "test",
@@ -214,6 +217,7 @@ lazy val scalding = Project(
  .aggregate(
   scaldingArgs,
   scaldingDate,
+  scaldingQuotation,
   scaldingCore,
   scaldingCommons,
   scaldingAvro,
@@ -242,6 +246,7 @@ lazy val scaldingAssembly = Project(
  .aggregate(
   scaldingArgs,
   scaldingDate,
+  scaldingQuotation,
   scaldingCore,
   scaldingCommons,
   scaldingAvro,
@@ -298,10 +303,8 @@ lazy val scaldingArgs = module("args")
 
 lazy val scaldingDate = module("date")
 
-lazy val scaldingGraph = module("graph")
-
 lazy val cascadingVersion =
-  System.getenv.asScala.getOrElse("SCALDING_CASCADING_VERSION", "3.2.1")
+  System.getenv.asScala.getOrElse("SCALDING_CASCADING_VERSION", "3.3.0-wip-18")
 
 lazy val cascadingJDBCVersion =
   System.getenv.asScala.getOrElse("SCALDING_CASCADING_JDBC_VERSION", "3.0.0-wip-127")
@@ -316,11 +319,19 @@ lazy val scaldingBenchmarks = module("benchmarks")
     parallelExecution in Test := false
   ).dependsOn(scaldingCore)
 
+lazy val scaldingQuotation = module("quotation").settings(
+  libraryDependencies ++= Seq(
+    "org.scala-lang" % "scala-reflect" % scalaVersion.value % "provided",
+    "org.scala-lang" % "scala-compiler" % scalaVersion.value % "provided"
+  )
+)
+
 lazy val scaldingCore = module("core").settings(
   libraryDependencies ++= Seq(
     "cascading" % "cascading-core" % cascadingVersion,
     "cascading" % "cascading-hadoop" % cascadingVersion,
     "cascading" % "cascading-local" % cascadingVersion,
+    "com.stripe" %% "dagon-core" % dagonVersion,
     "com.twitter" % "chill-hadoop" % chillVersion,
     "com.twitter" % "chill-java" % chillVersion,
     "com.twitter" %% "chill-bijection" % chillVersion,
@@ -337,7 +348,7 @@ lazy val scaldingCore = module("core").settings(
     "org.slf4j" % "slf4j-api" % slf4jVersion,
     "org.slf4j" % "slf4j-log4j12" % slf4jVersion % "provided"),
   addCompilerPlugin("org.scalamacros" % "paradise" % paradiseVersion cross CrossVersion.full)
-).dependsOn(scaldingArgs, scaldingDate, scaldingSerialization, maple)
+).dependsOn(scaldingArgs, scaldingDate, scaldingSerialization, maple, scaldingQuotation)
 
 lazy val scaldingCommons = module("commons").settings(
   libraryDependencies ++= Seq(

project/plugins.sbt

@@ -18,4 +18,4 @@ addSbtPlugin("com.typesafe.sbt"   % "sbt-git"             % "0.6.2")
 addSbtPlugin("com.typesafe.sbt"   % "sbt-scalariform"     % "1.3.0")
 addSbtPlugin("org.scoverage"      % "sbt-scoverage"       % "1.5.0")
 addSbtPlugin("org.xerial.sbt"     % "sbt-sonatype"        % "1.0")
-addSbtPlugin("org.wartremover"    % "sbt-wartremover"     % "2.0.2")
+addSbtPlugin("org.wartremover"    % "sbt-wartremover"     % "2.1.1")

scalding-commons/src/main/scala/com/twitter/scalding/commons/source/DailySources.scala

@@ -42,7 +42,7 @@ abstract class DailySuffixLzoProtobuf[T <: Message: Manifest](prefix: String, da
 
 abstract class DailySuffixMostRecentLzoProtobuf[T <: Message: Manifest](prefix: String, dateRange: DateRange)
   extends DailySuffixMostRecentSource(prefix, dateRange) with LzoProtobuf[T] {
-  override def column = manifest[T].erasure
+  override def column = manifest[T].runtimeClass
 }
 
 abstract class DailySuffixLzoThrift[T <: TBase[_, _]: Manifest](prefix: String, dateRange: DateRange)

scalding-commons/src/main/scala/com/twitter/scalding/commons/source/VersionedKeyValSource.scala

@@ -127,22 +127,20 @@ class VersionedKeyValSource[K, V](val path: String, val sourceVersion: Option[Lo
     }
 
   def sinkExists(mode: Mode): Boolean =
-    sinkVersion match {
-      case Some(version) =>
-        mode match {
-          case Test(buffers) =>
-            buffers(this) map { !_.isEmpty } getOrElse false
+    sinkVersion.exists { version =>
+      mode match {
+        case Test(buffers) =>
+          buffers(this) map { !_.isEmpty } getOrElse false
 
-          case HadoopTest(conf, buffers) =>
-            buffers(this) map { !_.isEmpty } getOrElse false
+        case HadoopTest(conf, buffers) =>
+          buffers(this) map { !_.isEmpty } getOrElse false
 
-          case m: HadoopMode =>
-            val conf = new JobConf(m.jobConf)
-            val store = sink.getStore(conf)
-            store.hasVersion(version)
-          case _ => sys.error(s"Unknown mode $mode")
-        }
-      case None => false
+        case m: HadoopMode =>
+          val conf = new JobConf(m.jobConf)
+          val store = sink.getStore(conf)
+          store.hasVersion(version)
+        case _ => sys.error(s"Unknown mode $mode")
+      }
     }
 
   override def createTap(readOrWrite: AccessMode)(implicit mode: Mode): Tap[_, _, _] = {

scalding-core/src/main/scala/com/twitter/package.scala

@@ -37,7 +37,7 @@ package object scalding {
   val scaldingVersion: String = "0.17.2"
 
   object RichPathFilter {
-    implicit def toRichPathFilter(f: PathFilter) = new RichPathFilter(f)
+    implicit def toRichPathFilter(f: PathFilter): RichPathFilter = new RichPathFilter(f)
   }
 
   class RichPathFilter(f: PathFilter) {

scalding-core/src/main/scala/com/twitter/scalding/ArgHelp.scala

@@ -5,10 +5,10 @@ sealed trait DescribedArg {
   def description: String
 }
 
-case class RequiredArg(key: String, description: String) extends DescribedArg
-case class OptionalArg(key: String, description: String) extends DescribedArg
-case class ListArg(key: String, description: String) extends DescribedArg
-case class BooleanArg(key: String, description: String) extends DescribedArg
+final case class RequiredArg(key: String, description: String) extends DescribedArg
+final case class OptionalArg(key: String, description: String) extends DescribedArg
+final case class ListArg(key: String, description: String) extends DescribedArg
+final case class BooleanArg(key: String, description: String) extends DescribedArg
 
 class HelpException extends RuntimeException("User asked for help")
 class DescriptionValidationException(msg: String) extends RuntimeException(msg)
@@ -119,4 +119,4 @@ trait ArgHelper {
   }
 }
 
-object ArgHelp extends ArgHelper
+object ArgHelp extends ArgHelper