From 5f56f829a0e190c7206c29fac9ca70447e7a9ea6 Mon Sep 17 00:00:00 2001 From: Matthieu Martin Date: Sat, 28 Mar 2015 09:20:10 -0700 Subject: [PATCH 1/6] Initial implicits and tests for basic HList integration. --- project/Build.scala | 12 +++++ .../com/twitter/scalding/TupleConverter.scala | 50 +++++++++++++++++++ .../com/twitter/scalding/TupleSetter.scala | 20 +++++++- .../com/twitter/scalding/TupleTest.scala | 46 +++++++++++++++++ 4 files changed, 127 insertions(+), 1 deletion(-) diff --git a/project/Build.scala b/project/Build.scala index d3e4a80a88..c5ea9fc73f 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -39,6 +39,7 @@ object ScaldingBuild extends Build { val scalaTestVersion = "2.2.2" val scalameterVersion = "0.6" val scroogeVersion = "3.17.0" + val shapelessVersion = "2.1.0" val slf4jVersion = "1.6.6" val thriftVersion = "0.5.0" @@ -269,6 +270,17 @@ object ScaldingBuild extends Build { "org.apache.hadoop" % "hadoop-core" % hadoopVersion % "provided", "org.slf4j" % "slf4j-api" % slf4jVersion, "org.slf4j" % "slf4j-log4j12" % slf4jVersion % "provided" + ) ++ ( + if(isScala210x(scalaVersion.value)) { + Seq( + "com.chuusai" % "shapeless_2.10.4" % shapelessVersion, + compilerPlugin("org.scalamacros" % "paradise_2.10.4" % "2.0.1") + ) + } else { + Seq( + "com.chuusai" %% "shapeless" % shapelessVersion + ) + } ) ).dependsOn(scaldingArgs, scaldingDate, maple) diff --git a/scalding-core/src/main/scala/com/twitter/scalding/TupleConverter.scala b/scalding-core/src/main/scala/com/twitter/scalding/TupleConverter.scala index 0a0f2c12d9..f3b723bb68 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/TupleConverter.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/TupleConverter.scala @@ -67,6 +67,56 @@ object TupleConverter extends GeneratedTupleConverters { def arity[T](implicit tc: TupleConverter[T]): Int = tc.arity def of[T](implicit tc: TupleConverter[T]): TupleConverter[T] = tc + import shapeless._ + import shapeless.ops.hlist._ + import shapeless.ops.nat._ + + type Aux[L <: HList, N <: Nat] = TupleConverter[L] { type Index = N } + + implicit def baseHListConverter[H, I <: Nat](implicit gH: TupleGetter[H], ti: ToInt[I]): Aux[H :: HNil, I] = + new TupleConverter[H :: HNil] { + type Index = I + override def apply(te: TupleEntry): H :: HNil = gH.get(te.getTuple, ti()) :: HNil + + override def arity: Int = 1 + } + + implicit def recursiveHListConverter[H, T <: HList, N <: Nat, I <: Nat] + (implicit + gH: TupleGetter[H], + len: Length.Aux[H :: T, N], + tii: ToInt[I], + tin: ToInt[N], + tailConverter: Aux[T, Succ[I]]): Aux[H :: T, I] = + new TupleConverter[H :: T] { + type Index = I + override def apply(te: TupleEntry): H :: T = + gH.get(te.getTuple, tii()) :: tailConverter(te) + + override def arity: Int = tin() + } + + implicit def initialRecursiveHListConverter[H, T <: HList, N <: Nat] + (implicit + gH: TupleGetter[H], + len: Length.Aux[H :: T, N], + ti: ToInt[N], + tailConverter: Aux[T, Nat._1]): TupleConverter[H :: T] = + new TupleConverter[H :: T] { + override def apply(te: TupleEntry): H :: T = + gH.get(te.getTuple, 0) :: tailConverter(te) + + override def arity: Int = ti() + } + + // Special case for a single element HList + implicit def singleElemHListConverter[H](implicit gH: TupleGetter[H]): TupleConverter[H :: HNil] = + new TupleConverter[H :: HNil] { + override def apply(te: TupleEntry): ::[H, HNil] = gH.get(te.getTuple, 0) :: HNil + + override def arity: Int = 1 + } + /** * Copies the tupleEntry, since cascading may change it after the end of an * operation (and it is not safe to assume the consumer has not kept a ref diff --git a/scalding-core/src/main/scala/com/twitter/scalding/TupleSetter.scala b/scalding-core/src/main/scala/com/twitter/scalding/TupleSetter.scala index f822b6b271..16ada6b1f9 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/TupleSetter.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/TupleSetter.scala @@ -16,7 +16,7 @@ limitations under the License. package com.twitter.scalding -import cascading.tuple.{ Tuple => CTuple } +import cascading.tuple.{Tuple => CTuple} /** * Typeclass to represent converting back to (setting into) a cascading Tuple @@ -65,6 +65,24 @@ object TupleSetter extends GeneratedTupleSetters { def arity[T](implicit ts: TupleSetter[T]): Int = ts.arity def of[T](implicit ts: TupleSetter[T]): TupleSetter[T] = ts + + import shapeless.ops.hlist._ + import shapeless.ops.nat._ + import shapeless._ + + implicit def hListSetter[L <: HList, N <: Nat] + (implicit tl: ToList[L, Any], len: Length.Aux[L, N], ti: ToInt[N]): + TupleSetter[L] = + new TupleSetter[L] { + + override def apply(arg: L): CTuple = { + val argList: List[Any] = arg.toList + new CTuple(argList.asInstanceOf[List[Object]]:_*) + } + + override def arity: Int = ti() + } + //This is here for handling functions that return cascading tuples: implicit lazy val CTupleSetter: TupleSetter[CTuple] = new TupleSetter[CTuple] { override def apply(arg: CTuple) = new CTuple(arg) diff --git a/scalding-core/src/test/scala/com/twitter/scalding/TupleTest.scala b/scalding-core/src/test/scala/com/twitter/scalding/TupleTest.scala index 7d3b2f2ac1..4a9e13a32a 100644 --- a/scalding-core/src/test/scala/com/twitter/scalding/TupleTest.scala +++ b/scalding-core/src/test/scala/com/twitter/scalding/TupleTest.scala @@ -18,6 +18,7 @@ package com.twitter.scalding import cascading.tuple.{ TupleEntry, Tuple => CTuple } import org.scalatest.{ Matchers, WordSpec } +import shapeless._ class TupleTest extends WordSpec with Matchers { def get[T](ctup: CTuple)(implicit tc: TupleConverter[T]) = tc(new TupleEntry(ctup)) @@ -62,6 +63,23 @@ class TupleTest extends WordSpec with Matchers { arityConvMatches((2, 3), 2) shouldBe true aritySetMatches((2, 3), 2) shouldBe true } + "get primitives out of cascading tuples using HLists" in { + val ctup = new CTuple("hey", new java.lang.Long(2), new java.lang.Integer(3)) + get[String :: Long :: Int :: HNil](ctup) shouldBe "hey" :: 2L :: 3 :: HNil + + roundTrip[Int :: HNil](3 :: HNil) shouldBe true + arityConvMatches(3 :: HNil, 1) shouldBe true + aritySetMatches(3 :: HNil, 1) shouldBe true + roundTrip[Long :: HNil](42L :: HNil) shouldBe true + arityConvMatches(42L :: HNil, 1) shouldBe true + aritySetMatches(42L :: HNil, 1) shouldBe true + roundTrip[String :: HNil]("hey" :: HNil) shouldBe true + arityConvMatches("hey" :: HNil, 1) shouldBe true + aritySetMatches("hey" :: HNil, 1) shouldBe true + roundTrip[Int :: Int :: HNil](4 :: 2 :: HNil) shouldBe true + arityConvMatches(2 :: 3 :: HNil, 2) shouldBe true + aritySetMatches(2 :: 3 :: HNil, 2) shouldBe true + } "get non-primitives out of cascading tuples" in { val ctup = new CTuple(None, List(1, 2, 3), 1 -> 2) get[(Option[Int], List[Int], (Int, Int))](ctup) shouldBe (None, List(1, 2, 3), 1 -> 2) @@ -75,6 +93,19 @@ class TupleTest extends WordSpec with Matchers { arityConvMatches(List(1, 2, 3), 1) shouldBe true aritySetMatches(List(1, 2, 3), 1) shouldBe true } + "get non-primitives out of cascading tuples using HLists" in { + val ctup = new CTuple(None, List(1, 2, 3), 1 -> 2) + get[Option[Int] :: List[Int] :: (Int, Int) :: HNil](ctup) shouldBe None :: List(1, 2, 3) :: 1 -> 2 :: HNil + + roundTrip[Option[Int] :: List[Int] :: HNil](Some(1) :: List() :: HNil) shouldBe true + arityConvMatches(None :: Nil :: HNil, 2) shouldBe true + aritySetMatches(None :: Nil :: HNil, 2) shouldBe true + + arityConvMatches(None :: HNil, 1) shouldBe true + aritySetMatches(None :: HNil, 1) shouldBe true + arityConvMatches(List(1, 2, 3) :: HNil, 1) shouldBe true + aritySetMatches(List(1, 2, 3) :: HNil, 1) shouldBe true + } "deal with AnyRef" in { val ctup = new CTuple(None, List(1, 2, 3), 1 -> 2) get[(AnyRef, AnyRef, AnyRef)](ctup) shouldBe (None, List(1, 2, 3), 1 -> 2) @@ -85,5 +116,20 @@ class TupleTest extends WordSpec with Matchers { arityConvMatches[(AnyRef, AnyRef)](("hey", "you"), 2) shouldBe true aritySetMatches[(AnyRef, AnyRef)](("hey", "you"), 2) shouldBe true } + "deal with AnyRef using HLists" in { + val ctup = new CTuple(None, List(1, 2, 3), 1 -> 2) + get[AnyRef :: AnyRef :: AnyRef :: HNil](ctup) shouldBe None :: List(1, 2, 3) :: 1 -> 2 :: HNil + // TODO: uncomment the expectation below + // for some reason the implicit for this case is not being picked up and the LowPriorityTupleConverters is used + // instead which results in the following failed expectation: + // + // "you" was not equal to you :: HNil + //get[AnyRef :: HNil](new CTuple("you")) shouldBe "you" :: HNil + + roundTrip[AnyRef :: HNil]("hey" :: HNil) shouldBe true + roundTrip[AnyRef :: AnyRef :: HNil](Nil :: Nil :: HNil) shouldBe true + arityConvMatches[AnyRef :: AnyRef :: HNil]("hey" :: "you" :: HNil, 2) shouldBe true + aritySetMatches[AnyRef :: AnyRef :: HNil]("hey" :: "you" :: HNil, 2) shouldBe true + } } } From ee0f899d8d3bb89dd6a88eea662d5d434305bfde Mon Sep 17 00:00:00 2001 From: Matthieu Martin Date: Sat, 28 Mar 2015 20:06:24 -0700 Subject: [PATCH 2/6] All tests are passing now. --- project/Build.scala | 3 +- .../com/twitter/scalding/TupleConverter.scala | 88 +++++++++---------- .../com/twitter/scalding/TupleSetter.scala | 13 ++- .../com/twitter/scalding/TupleTest.scala | 7 +- 4 files changed, 50 insertions(+), 61 deletions(-) diff --git a/project/Build.scala b/project/Build.scala index c5ea9fc73f..7a2fef4fd5 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -273,8 +273,7 @@ object ScaldingBuild extends Build { ) ++ ( if(isScala210x(scalaVersion.value)) { Seq( - "com.chuusai" % "shapeless_2.10.4" % shapelessVersion, - compilerPlugin("org.scalamacros" % "paradise_2.10.4" % "2.0.1") + "com.chuusai" % "shapeless_2.10.4" % shapelessVersion ) } else { Seq( diff --git a/scalding-core/src/main/scala/com/twitter/scalding/TupleConverter.scala b/scalding-core/src/main/scala/com/twitter/scalding/TupleConverter.scala index f3b723bb68..3114e92304 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/TupleConverter.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/TupleConverter.scala @@ -18,8 +18,9 @@ package com.twitter.scalding import cascading.tuple.TupleEntry import cascading.tuple.{ Tuple => CTuple } +import shapeless._ -import scala.collection.breakOut +import scala.collection.{GenTraversable, breakOut} /** * Typeclass to represent converting from cascading TupleEntry to some type T. @@ -51,6 +52,37 @@ trait LowPriorityTupleConverters extends java.io.Serializable { } } +/** + * Based on `FromTraversable` from shapeless + */ +trait FromTraversable[Out <: HList] { + def apply(l : GenTraversable[_]) : Option[Out] +} + +object FromTraversable { + def apply[Out <: HList](implicit from: FromTraversable[Out]) = from + + implicit def hnilFromTraversable[T] = new FromTraversable[HNil] { + def apply(l : GenTraversable[_]) = + if(l.isEmpty) Some(HNil) else None + } + + implicit def hlistFromTraversable[OutH, OutT <: HList] + (implicit flt : FromTraversable[OutT], g : TupleGetter[OutH]) = new FromTraversable[OutH :: OutT] { + def apply(l : GenTraversable[_]) : Option[OutH :: OutT] = + if(l.isEmpty) None + else for( + h <- new Some( + g.get( + new CTuple(l.head.asInstanceOf[Object]), + 0 + ) + ); + t <- flt(l.tail) + ) yield h :: t + } +} + object TupleConverter extends GeneratedTupleConverters { /** * Treat this TupleConverter as one for a superclass @@ -67,56 +99,22 @@ object TupleConverter extends GeneratedTupleConverters { def arity[T](implicit tc: TupleConverter[T]): Int = tc.arity def of[T](implicit tc: TupleConverter[T]): TupleConverter[T] = tc - import shapeless._ import shapeless.ops.hlist._ import shapeless.ops.nat._ - type Aux[L <: HList, N <: Nat] = TupleConverter[L] { type Index = N } - - implicit def baseHListConverter[H, I <: Nat](implicit gH: TupleGetter[H], ti: ToInt[I]): Aux[H :: HNil, I] = - new TupleConverter[H :: HNil] { - type Index = I - override def apply(te: TupleEntry): H :: HNil = gH.get(te.getTuple, ti()) :: HNil - - override def arity: Int = 1 - } - - implicit def recursiveHListConverter[H, T <: HList, N <: Nat, I <: Nat] - (implicit - gH: TupleGetter[H], - len: Length.Aux[H :: T, N], - tii: ToInt[I], - tin: ToInt[N], - tailConverter: Aux[T, Succ[I]]): Aux[H :: T, I] = + implicit def hListConverter[H, T <: HList, N <: Nat] + (implicit g: TupleGetter[H], len: Length.Aux[H :: T, N], toIntN : ToInt[N], fl : FromTraversable[H :: T]): TupleConverter[H :: T] = new TupleConverter[H :: T] { - type Index = I - override def apply(te: TupleEntry): H :: T = - gH.get(te.getTuple, tii()) :: tailConverter(te) - - override def arity: Int = tin() + import scala.collection.JavaConverters._ + override def apply(te: TupleEntry): H :: T = { + val genTraversable = te.getTupleCopy.asScala + val l : Option[H :: T] = fl(genTraversable) + l.get + } + + override def arity: Int = toIntN() } - implicit def initialRecursiveHListConverter[H, T <: HList, N <: Nat] - (implicit - gH: TupleGetter[H], - len: Length.Aux[H :: T, N], - ti: ToInt[N], - tailConverter: Aux[T, Nat._1]): TupleConverter[H :: T] = - new TupleConverter[H :: T] { - override def apply(te: TupleEntry): H :: T = - gH.get(te.getTuple, 0) :: tailConverter(te) - - override def arity: Int = ti() - } - - // Special case for a single element HList - implicit def singleElemHListConverter[H](implicit gH: TupleGetter[H]): TupleConverter[H :: HNil] = - new TupleConverter[H :: HNil] { - override def apply(te: TupleEntry): ::[H, HNil] = gH.get(te.getTuple, 0) :: HNil - - override def arity: Int = 1 - } - /** * Copies the tupleEntry, since cascading may change it after the end of an * operation (and it is not safe to assume the consumer has not kept a ref diff --git a/scalding-core/src/main/scala/com/twitter/scalding/TupleSetter.scala b/scalding-core/src/main/scala/com/twitter/scalding/TupleSetter.scala index 16ada6b1f9..8f12fedd93 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/TupleSetter.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/TupleSetter.scala @@ -16,7 +16,8 @@ limitations under the License. package com.twitter.scalding -import cascading.tuple.{Tuple => CTuple} +import cascading.tuple.{ Tuple => CTuple } +import shapeless.PolyDefns.~> /** * Typeclass to represent converting back to (setting into) a cascading Tuple @@ -65,19 +66,15 @@ object TupleSetter extends GeneratedTupleSetters { def arity[T](implicit ts: TupleSetter[T]): Int = ts.arity def of[T](implicit ts: TupleSetter[T]): TupleSetter[T] = ts - import shapeless.ops.hlist._ import shapeless.ops.nat._ import shapeless._ - implicit def hListSetter[L <: HList, N <: Nat] - (implicit tl: ToList[L, Any], len: Length.Aux[L, N], ti: ToInt[N]): - TupleSetter[L] = - new TupleSetter[L] { + implicit def hListSetter[L <: HList, N <: Nat, T <: Any](implicit len: Length.Aux[L, N], ti: ToInt[N], tl: ToList[L, T]) = new TupleSetter[L] { override def apply(arg: L): CTuple = { - val argList: List[Any] = arg.toList - new CTuple(argList.asInstanceOf[List[Object]]:_*) + val list = arg.toList[T].map(_.asInstanceOf[Object]) + new CTuple(list:_*) } override def arity: Int = ti() diff --git a/scalding-core/src/test/scala/com/twitter/scalding/TupleTest.scala b/scalding-core/src/test/scala/com/twitter/scalding/TupleTest.scala index 4a9e13a32a..b9af9d35b0 100644 --- a/scalding-core/src/test/scala/com/twitter/scalding/TupleTest.scala +++ b/scalding-core/src/test/scala/com/twitter/scalding/TupleTest.scala @@ -119,12 +119,7 @@ class TupleTest extends WordSpec with Matchers { "deal with AnyRef using HLists" in { val ctup = new CTuple(None, List(1, 2, 3), 1 -> 2) get[AnyRef :: AnyRef :: AnyRef :: HNil](ctup) shouldBe None :: List(1, 2, 3) :: 1 -> 2 :: HNil - // TODO: uncomment the expectation below - // for some reason the implicit for this case is not being picked up and the LowPriorityTupleConverters is used - // instead which results in the following failed expectation: - // - // "you" was not equal to you :: HNil - //get[AnyRef :: HNil](new CTuple("you")) shouldBe "you" :: HNil + get[AnyRef :: HNil](new CTuple("you")) shouldBe "you" :: HNil roundTrip[AnyRef :: HNil]("hey" :: HNil) shouldBe true roundTrip[AnyRef :: AnyRef :: HNil](Nil :: Nil :: HNil) shouldBe true From 4af19fb5e4e488611c68aa64be7184bc26dd24c3 Mon Sep 17 00:00:00 2001 From: Matthieu Martin Date: Sat, 28 Mar 2015 20:19:56 -0700 Subject: [PATCH 3/6] Some cleanup/formatting. --- .../com/twitter/scalding/TupleConverter.scala | 10 +++++++--- .../scala/com/twitter/scalding/TupleSetter.scala | 14 +++++++------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/scalding-core/src/main/scala/com/twitter/scalding/TupleConverter.scala b/scalding-core/src/main/scala/com/twitter/scalding/TupleConverter.scala index 3114e92304..f2161aa8d4 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/TupleConverter.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/TupleConverter.scala @@ -103,12 +103,16 @@ object TupleConverter extends GeneratedTupleConverters { import shapeless.ops.nat._ implicit def hListConverter[H, T <: HList, N <: Nat] - (implicit g: TupleGetter[H], len: Length.Aux[H :: T, N], toIntN : ToInt[N], fl : FromTraversable[H :: T]): TupleConverter[H :: T] = + (implicit + g: TupleGetter[H], + len: Length.Aux[H :: T, N], + toIntN : ToInt[N], + fl : FromTraversable[H :: T]): TupleConverter[H :: T] = new TupleConverter[H :: T] { import scala.collection.JavaConverters._ override def apply(te: TupleEntry): H :: T = { - val genTraversable = te.getTupleCopy.asScala - val l : Option[H :: T] = fl(genTraversable) + val iterable = te.getTupleCopy.asScala + val l : Option[H :: T] = fl(iterable) l.get } diff --git a/scalding-core/src/main/scala/com/twitter/scalding/TupleSetter.scala b/scalding-core/src/main/scala/com/twitter/scalding/TupleSetter.scala index 8f12fedd93..1ca71cb43b 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/TupleSetter.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/TupleSetter.scala @@ -17,7 +17,6 @@ limitations under the License. package com.twitter.scalding import cascading.tuple.{ Tuple => CTuple } -import shapeless.PolyDefns.~> /** * Typeclass to represent converting back to (setting into) a cascading Tuple @@ -70,15 +69,16 @@ object TupleSetter extends GeneratedTupleSetters { import shapeless.ops.nat._ import shapeless._ - implicit def hListSetter[L <: HList, N <: Nat, T <: Any](implicit len: Length.Aux[L, N], ti: ToInt[N], tl: ToList[L, T]) = new TupleSetter[L] { + implicit def hListSetter[L <: HList, N <: Nat, T <: Any] + (implicit len: Length.Aux[L, N], ti: ToInt[N], tl: ToList[L, T]) = new TupleSetter[L] { - override def apply(arg: L): CTuple = { + override def apply(arg: L): CTuple = { val list = arg.toList[T].map(_.asInstanceOf[Object]) - new CTuple(list:_*) - } + new CTuple(list:_*) + } - override def arity: Int = ti() - } + override def arity: Int = ti() + } //This is here for handling functions that return cascading tuples: implicit lazy val CTupleSetter: TupleSetter[CTuple] = new TupleSetter[CTuple] { From 2432bc6f5cd073960a29cdcc2b50919a57a4b7c0 Mon Sep 17 00:00:00 2001 From: Matthieu Martin Date: Sat, 28 Mar 2015 20:28:32 -0700 Subject: [PATCH 4/6] another formatting change --- .../com/twitter/scalding/TupleConverter.scala | 34 ++++++++----------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/scalding-core/src/main/scala/com/twitter/scalding/TupleConverter.scala b/scalding-core/src/main/scala/com/twitter/scalding/TupleConverter.scala index f2161aa8d4..f85b26ba2a 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/TupleConverter.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/TupleConverter.scala @@ -55,32 +55,28 @@ trait LowPriorityTupleConverters extends java.io.Serializable { /** * Based on `FromTraversable` from shapeless */ -trait FromTraversable[Out <: HList] { +trait FromTraversableWithTupleGetter[Out <: HList] { def apply(l : GenTraversable[_]) : Option[Out] } -object FromTraversable { - def apply[Out <: HList](implicit from: FromTraversable[Out]) = from +object FromTraversableWithTupleGetter { + def apply[Out <: HList](implicit from: FromTraversableWithTupleGetter[Out]) = from - implicit def hnilFromTraversable[T] = new FromTraversable[HNil] { + implicit def hnilFromTraversableWithTupleGetter[T] = new FromTraversableWithTupleGetter[HNil] { def apply(l : GenTraversable[_]) = if(l.isEmpty) Some(HNil) else None } - implicit def hlistFromTraversable[OutH, OutT <: HList] - (implicit flt : FromTraversable[OutT], g : TupleGetter[OutH]) = new FromTraversable[OutH :: OutT] { - def apply(l : GenTraversable[_]) : Option[OutH :: OutT] = - if(l.isEmpty) None - else for( - h <- new Some( - g.get( - new CTuple(l.head.asInstanceOf[Object]), - 0 - ) - ); - t <- flt(l.tail) - ) yield h :: t - } + implicit def hlistFromTraversableWithTupleGetter[OutH, OutT <: HList] + (implicit flt : FromTraversableWithTupleGetter[OutT], g : TupleGetter[OutH]) = + new FromTraversableWithTupleGetter[OutH :: OutT] { + def apply(l : GenTraversable[_]) : Option[OutH :: OutT] = + if(l.isEmpty) None + else for( + h <- new Some(g.get(new CTuple(l.head.asInstanceOf[Object]), 0)); + t <- flt(l.tail) + ) yield h :: t + } } object TupleConverter extends GeneratedTupleConverters { @@ -107,7 +103,7 @@ object TupleConverter extends GeneratedTupleConverters { g: TupleGetter[H], len: Length.Aux[H :: T, N], toIntN : ToInt[N], - fl : FromTraversable[H :: T]): TupleConverter[H :: T] = + fl : FromTraversableWithTupleGetter[H :: T]): TupleConverter[H :: T] = new TupleConverter[H :: T] { import scala.collection.JavaConverters._ override def apply(te: TupleEntry): H :: T = { From 0631ac7ee0dac0d82a8bd0d627d76f424a8e6602 Mon Sep 17 00:00:00 2001 From: Matthieu Martin Date: Sat, 28 Mar 2015 20:50:05 -0700 Subject: [PATCH 5/6] Simplify the TupleConverter a bit --- .../com/twitter/scalding/TupleConverter.scala | 42 ++++++++++--------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/scalding-core/src/main/scala/com/twitter/scalding/TupleConverter.scala b/scalding-core/src/main/scala/com/twitter/scalding/TupleConverter.scala index f85b26ba2a..ad2b92f8a8 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/TupleConverter.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/TupleConverter.scala @@ -19,6 +19,7 @@ package com.twitter.scalding import cascading.tuple.TupleEntry import cascading.tuple.{ Tuple => CTuple } import shapeless._ +import shapeless.ops.nat._ import scala.collection.{GenTraversable, breakOut} @@ -55,26 +56,30 @@ trait LowPriorityTupleConverters extends java.io.Serializable { /** * Based on `FromTraversable` from shapeless */ -trait FromTraversableWithTupleGetter[Out <: HList] { - def apply(l : GenTraversable[_]) : Option[Out] +trait FromTupleEntry[Out <: HList, I <: Nat] { + def apply(l : TupleEntry) : Option[Out] } -object FromTraversableWithTupleGetter { - def apply[Out <: HList](implicit from: FromTraversableWithTupleGetter[Out]) = from +object FromTupleEntry { + def apply[Out <: HList](implicit from: FromTupleEntry[Out, Nat._0]) = from - implicit def hnilFromTraversableWithTupleGetter[T] = new FromTraversableWithTupleGetter[HNil] { - def apply(l : GenTraversable[_]) = - if(l.isEmpty) Some(HNil) else None - } + implicit def hnilFromTupleEntry[T, I <: Nat](implicit toIntI : ToInt[I]) = + new FromTupleEntry[HNil, I] { + def apply(te : TupleEntry) = + if(te.size() == toIntI()) Some(HNil) else None + } - implicit def hlistFromTraversableWithTupleGetter[OutH, OutT <: HList] - (implicit flt : FromTraversableWithTupleGetter[OutT], g : TupleGetter[OutH]) = - new FromTraversableWithTupleGetter[OutH :: OutT] { - def apply(l : GenTraversable[_]) : Option[OutH :: OutT] = - if(l.isEmpty) None + implicit def hlistFromTupleEntry[OutH, OutT <: HList, I <: Nat] + (implicit + flt : FromTupleEntry[OutT, Succ[I]], + g : TupleGetter[OutH], + toIntI : ToInt[I]) = + new FromTupleEntry[OutH :: OutT, I] { + def apply(te : TupleEntry) : Option[OutH :: OutT] = + if(te.size() <= toIntI()) None else for( - h <- new Some(g.get(new CTuple(l.head.asInstanceOf[Object]), 0)); - t <- flt(l.tail) + h <- new Some(g.get(te.getTuple, toIntI())); + t <- flt(te) ) yield h :: t } } @@ -96,19 +101,16 @@ object TupleConverter extends GeneratedTupleConverters { def of[T](implicit tc: TupleConverter[T]): TupleConverter[T] = tc import shapeless.ops.hlist._ - import shapeless.ops.nat._ implicit def hListConverter[H, T <: HList, N <: Nat] (implicit g: TupleGetter[H], len: Length.Aux[H :: T, N], toIntN : ToInt[N], - fl : FromTraversableWithTupleGetter[H :: T]): TupleConverter[H :: T] = + fl : FromTupleEntry[H :: T, Nat._0]): TupleConverter[H :: T] = new TupleConverter[H :: T] { - import scala.collection.JavaConverters._ override def apply(te: TupleEntry): H :: T = { - val iterable = te.getTupleCopy.asScala - val l : Option[H :: T] = fl(iterable) + val l : Option[H :: T] = fl(te) l.get } From ab18131c387da9e3d8670a1096c048670840ee51 Mon Sep 17 00:00:00 2001 From: Matthieu Martin Date: Sat, 28 Mar 2015 22:32:04 -0700 Subject: [PATCH 6/6] Fix spacing --- .../src/main/scala/com/twitter/scalding/TupleSetter.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scalding-core/src/main/scala/com/twitter/scalding/TupleSetter.scala b/scalding-core/src/main/scala/com/twitter/scalding/TupleSetter.scala index 1ca71cb43b..1329b02a5c 100644 --- a/scalding-core/src/main/scala/com/twitter/scalding/TupleSetter.scala +++ b/scalding-core/src/main/scala/com/twitter/scalding/TupleSetter.scala @@ -73,7 +73,7 @@ object TupleSetter extends GeneratedTupleSetters { (implicit len: Length.Aux[L, N], ti: ToInt[N], tl: ToList[L, T]) = new TupleSetter[L] { override def apply(arg: L): CTuple = { - val list = arg.toList[T].map(_.asInstanceOf[Object]) + val list = arg.toList[T].map(_.asInstanceOf[Object]) new CTuple(list:_*) }