From 8d9efaf0352c0ca86bc01c73f72c8243fdfd44c6 Mon Sep 17 00:00:00 2001 From: Kenneth Gudel Date: Mon, 25 Apr 2016 11:41:20 -0400 Subject: [PATCH 1/9] explicitly handle periods, as toInt fails with decimals --- .../grasshopper/geocoder/search/census/SearchUtils.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/geocoder/src/main/scala/grasshopper/geocoder/search/census/SearchUtils.scala b/geocoder/src/main/scala/grasshopper/geocoder/search/census/SearchUtils.scala index de02da8..3c1068c 100644 --- a/geocoder/src/main/scala/grasshopper/geocoder/search/census/SearchUtils.scala +++ b/geocoder/src/main/scala/grasshopper/geocoder/search/census/SearchUtils.scala @@ -12,7 +12,9 @@ object SearchUtils { if (isNumeric(s)) { Some(s.toInt) } else { - if (s.contains("-")) { + if (s.contains(".")) { + Some(s.substring(0, s.indexOf(".")).toInt) + } else if (s.contains("-")) { Some(s.substring(0, s.indexOf("-")).toInt) } else if (s == "None") { None @@ -28,6 +30,6 @@ object SearchUtils { } def isNumeric(str: String): Boolean = { - str.matches("-?\\d+(\\.\\d+)?") + str.matches("-?\\d+") } } From 0378656d0cfb5977d25b5231d6ebb59944c4702d Mon Sep 17 00:00:00 2001 From: Kenneth Gudel Date: Mon, 25 Apr 2016 11:50:07 -0400 Subject: [PATCH 2/9] would currently return None when encountering a negative number with a hyphen the change makes the code ignore a hyphen as the first digit to get around this --- .../scala/grasshopper/geocoder/search/census/SearchUtils.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geocoder/src/main/scala/grasshopper/geocoder/search/census/SearchUtils.scala b/geocoder/src/main/scala/grasshopper/geocoder/search/census/SearchUtils.scala index 3c1068c..f676746 100644 --- a/geocoder/src/main/scala/grasshopper/geocoder/search/census/SearchUtils.scala +++ b/geocoder/src/main/scala/grasshopper/geocoder/search/census/SearchUtils.scala @@ -15,7 +15,7 @@ object SearchUtils { if (s.contains(".")) { Some(s.substring(0, s.indexOf(".")).toInt) } else if (s.contains("-")) { - Some(s.substring(0, s.indexOf("-")).toInt) + Some(s.substring(0, s.indexOf("-", 1)).toInt) } else if (s == "None") { None } else { From e456eac200902e8e32cf2629f68d871067eb418d Mon Sep 17 00:00:00 2001 From: Kenneth Gudel Date: Mon, 25 Apr 2016 11:51:28 -0400 Subject: [PATCH 3/9] reverting to scalaCheck 1.12.5 as property based checks fail with 1.13.0 --- project/Version.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Version.scala b/project/Version.scala index a6ff555..c384942 100644 --- a/project/Version.scala +++ b/project/Version.scala @@ -4,7 +4,7 @@ object Version { val config = "1.3.0" val akka = "2.4.4" val scalaTest = "3.0.0-M15" - val scalaCheck = "1.13.0" + val scalaCheck = "1.12.5" val elasticsearch = "2.2.0" val jts = "1.13" val scale = "0.0.2" From 18bb1ae03355375b6189631297e8a25942f39163 Mon Sep 17 00:00:00 2001 From: Kenneth Gudel Date: Mon, 25 Apr 2016 11:52:54 -0400 Subject: [PATCH 4/9] begin adding property tests for searchutils --- .../search/census/NumericGenerators.scala | 19 +++++++++++++ .../search/census/SearchUtilsSpec.scala | 28 +++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 geocoder/src/test/scala/grasshopper/geocoder/search/census/NumericGenerators.scala create mode 100644 geocoder/src/test/scala/grasshopper/geocoder/search/census/SearchUtilsSpec.scala diff --git a/geocoder/src/test/scala/grasshopper/geocoder/search/census/NumericGenerators.scala b/geocoder/src/test/scala/grasshopper/geocoder/search/census/NumericGenerators.scala new file mode 100644 index 0000000..a2b0233 --- /dev/null +++ b/geocoder/src/test/scala/grasshopper/geocoder/search/census/NumericGenerators.scala @@ -0,0 +1,19 @@ +package grasshopper.geocoder.search.census + +import org.scalacheck.Gen + +trait NumericGenerators { + + def digits: Gen[Int] = { + for { + n <- Gen.choose(-10000, 10000) + } yield n + } + + def positive: Gen[Int] = { + for { + n <- Gen.choose(0, 10000) + } yield n + } + +} \ No newline at end of file diff --git a/geocoder/src/test/scala/grasshopper/geocoder/search/census/SearchUtilsSpec.scala b/geocoder/src/test/scala/grasshopper/geocoder/search/census/SearchUtilsSpec.scala new file mode 100644 index 0000000..d12ba13 --- /dev/null +++ b/geocoder/src/test/scala/grasshopper/geocoder/search/census/SearchUtilsSpec.scala @@ -0,0 +1,28 @@ +package grasshopper.geocoder.search.census + +import org.scalatest.{ PropSpec, MustMatchers } +import org.scalatest.prop.PropertyChecks +import grasshopper.geocoder.search.census.SearchUtils._ + +class SearchUtilsSpec extends PropSpec with MustMatchers with PropertyChecks with NumericGenerators { + + property("digits are parsed into Ints") { + forAll(digits) { n => + toInt(n.toString) mustBe Some(n) + } + } + + property("decimals are parsed into Ints") { + forAll(digits, positive) { (x, y) => + val numString = x.toString + "." + y.toString + toInt(numString) mustBe Some(x) + } + } + + property("hyphens are parsed into Ints") { + forAll(digits, positive) { (x, y) => + val numString = x.toString + "-" + y.toString + toInt(numString) mustBe Some(x) + } + } +} From 7bac7a2632cba8a9e21e61dcd76c9f62fdcdf0eb Mon Sep 17 00:00:00 2001 From: Kenneth Gudel Date: Thu, 28 Apr 2016 14:13:04 -0400 Subject: [PATCH 5/9] testing whether the field has has no value seems like something that should be done by toInt. This produces fewer lines of code and seems more logically consistant, rather than breaking out one case for special treatment --- .../search/census/AddressInterpolator.scala | 21 +++++-------------- .../geocoder/search/census/SearchUtils.scala | 2 ++ 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/geocoder/src/main/scala/grasshopper/geocoder/search/census/AddressInterpolator.scala b/geocoder/src/main/scala/grasshopper/geocoder/search/census/AddressInterpolator.scala index 1fe76b3..3b9ea4a 100644 --- a/geocoder/src/main/scala/grasshopper/geocoder/search/census/AddressInterpolator.scala +++ b/geocoder/src/main/scala/grasshopper/geocoder/search/census/AddressInterpolator.scala @@ -17,14 +17,9 @@ object AddressInterpolator { val s = feature.values.getOrElse(s"${pre}FROMHN", "0") val e = feature.values.getOrElse(s"${pre}TOHN", "0") - val start = s match { - case "" => 0 - case _ => toInt(s.toString).getOrElse(0) - } - val end = e match { - case "" => 0 - case _ => toInt(e.toString).getOrElse(0) - } + val start = toInt(s.toString).getOrElse(0) + + val end = toInt(e.toString).getOrElse(0) AddressRange(start, end) } @@ -98,17 +93,11 @@ object AddressInterpolator { private def rangeIsEven(f: Feature, isLeft: Boolean): Boolean = { val prefix = if (isLeft) "L" else "R" val fromEven = f.values.getOrElse(s"${prefix}FROMHN", 0) - val fromRange = fromEven match { - case "" => 0 - case _ => toInt(fromEven.toString).getOrElse(0) - } + val fromRange = toInt(fromEven.toString).getOrElse(0) val fromIsEven = fromEven != "" && fromRange % 2 == 0 val toEven = f.values.getOrElse(s"${prefix}TOHN", 0) - val toRange = fromEven match { - case "" => 0 - case _ => toInt(toEven.toString).getOrElse(0) - } + val toRange = toInt(toEven.toString).getOrElse(0) val toIsEven = toEven != "" && toRange % 2 == 0 fromIsEven || toIsEven } diff --git a/geocoder/src/main/scala/grasshopper/geocoder/search/census/SearchUtils.scala b/geocoder/src/main/scala/grasshopper/geocoder/search/census/SearchUtils.scala index f676746..10d3f3a 100644 --- a/geocoder/src/main/scala/grasshopper/geocoder/search/census/SearchUtils.scala +++ b/geocoder/src/main/scala/grasshopper/geocoder/search/census/SearchUtils.scala @@ -18,6 +18,8 @@ object SearchUtils { Some(s.substring(0, s.indexOf("-", 1)).toInt) } else if (s == "None") { None + } else if (s == "") { + Some(0) } else { Some(s.replaceAll("[^\\d]", "").toInt) } From 95637ee421c612eb997686e341d324e68103dd0c Mon Sep 17 00:00:00 2001 From: Kenneth Gudel Date: Thu, 28 Apr 2016 14:17:21 -0400 Subject: [PATCH 6/9] switched no field to return None instead of 0 to make it more constant with no entry in field --- .../scala/grasshopper/geocoder/search/census/SearchUtils.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geocoder/src/main/scala/grasshopper/geocoder/search/census/SearchUtils.scala b/geocoder/src/main/scala/grasshopper/geocoder/search/census/SearchUtils.scala index 10d3f3a..8936f66 100644 --- a/geocoder/src/main/scala/grasshopper/geocoder/search/census/SearchUtils.scala +++ b/geocoder/src/main/scala/grasshopper/geocoder/search/census/SearchUtils.scala @@ -19,7 +19,7 @@ object SearchUtils { } else if (s == "None") { None } else if (s == "") { - Some(0) + None } else { Some(s.replaceAll("[^\\d]", "").toInt) } From 2c2212a563b7dfea20996d472382bd8fb7a8e71b Mon Sep 17 00:00:00 2001 From: Kenneth Gudel Date: Thu, 28 Apr 2016 14:23:15 -0400 Subject: [PATCH 7/9] removing the check for whether a field is blank as it seems arbitrary whether a blank field should be even or odd --- .../geocoder/search/census/AddressInterpolator.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/geocoder/src/main/scala/grasshopper/geocoder/search/census/AddressInterpolator.scala b/geocoder/src/main/scala/grasshopper/geocoder/search/census/AddressInterpolator.scala index 3b9ea4a..38403b0 100644 --- a/geocoder/src/main/scala/grasshopper/geocoder/search/census/AddressInterpolator.scala +++ b/geocoder/src/main/scala/grasshopper/geocoder/search/census/AddressInterpolator.scala @@ -94,11 +94,11 @@ object AddressInterpolator { val prefix = if (isLeft) "L" else "R" val fromEven = f.values.getOrElse(s"${prefix}FROMHN", 0) val fromRange = toInt(fromEven.toString).getOrElse(0) - val fromIsEven = fromEven != "" && fromRange % 2 == 0 + val fromIsEven = fromRange != None && fromRange % 2 == 0 val toEven = f.values.getOrElse(s"${prefix}TOHN", 0) val toRange = toInt(toEven.toString).getOrElse(0) - val toIsEven = toEven != "" && toRange % 2 == 0 + val toIsEven = toRange != None && toRange % 2 == 0 fromIsEven || toIsEven } From 6abb44c0edeec6f623c0614439a8ed7143cf7aa3 Mon Sep 17 00:00:00 2001 From: Kenneth Gudel Date: Thu, 28 Apr 2016 15:47:12 -0400 Subject: [PATCH 8/9] adding property tests for many of the functions in addressInterpolator, missing the full interpolator and prefix. Also, I believe all of the other tests are covered by the final two test which cover caluclateAddressRange, I was not sure if there were still value in the other tests so I am leaving them --- .../census/AddressInterpolatorSpec.scala | 84 +++++++++++++++++++ .../search/census/NumericGenerators.scala | 12 +++ 2 files changed, 96 insertions(+) create mode 100644 geocoder/src/test/scala/grasshopper/geocoder/search/census/AddressInterpolatorSpec.scala diff --git a/geocoder/src/test/scala/grasshopper/geocoder/search/census/AddressInterpolatorSpec.scala b/geocoder/src/test/scala/grasshopper/geocoder/search/census/AddressInterpolatorSpec.scala new file mode 100644 index 0000000..df4e38d --- /dev/null +++ b/geocoder/src/test/scala/grasshopper/geocoder/search/census/AddressInterpolatorSpec.scala @@ -0,0 +1,84 @@ +package grasshopper.geocoder.search.census + +import org.scalatest.{ PropSpec, MustMatchers, PrivateMethodTester } +import org.scalatest.prop.PropertyChecks +import grasshopper.geocoder.search.census.AddressInterpolator._ +import grasshopper.model.census.AddressRange +import feature._ + +class AddressInterpolatorSpec extends PropSpec with MustMatchers with PropertyChecks with PrivateMethodTester with NumericGenerators { + + property("calculateDistance must always be positive") { + forAll(positive, positive) { (start, stop) => + val range = new AddressRange(start, stop) + val calculateDistance = PrivateMethod[Double]('calculateDistance) + AddressInterpolator invokePrivate calculateDistance(range) mustBe >=(0.0) + } + } + + val testField = new Field("test", StringType()) + val testSchema = new Schema(Iterable(testField, testField)) + + property("for rangeIsEven left even is even") { + forAll(even, even) { (x, y) => + val range = Map("LFROMHN" -> x, "LTOHN" -> y) + val evenFeature = new Feature(1, testSchema, range) + val rangeIsEven = PrivateMethod[Boolean]('rangeIsEven) + AddressInterpolator invokePrivate rangeIsEven(evenFeature, true) mustBe true + } + } + + property("for rangeIsEven left odd is odd") { + forAll(odd, odd) { (x, y) => + whenever(x != 0 && y != 0) { + val range = Map("LFROMHN" -> x, "LTOHN" -> y) + val oddFeature = new Feature(1, testSchema, range) + val rangeIsEven = PrivateMethod[Boolean]('rangeIsEven) + AddressInterpolator invokePrivate rangeIsEven(oddFeature, true) mustBe false + } + } + } + + property("for rangeIsEven right even is even") { + forAll(even, even) { (x, y) => + val range = Map("RFROMHN" -> x, "RTOHN" -> y) + val evenFeature = new Feature(1, testSchema, range) + val rangeIsEven = PrivateMethod[Boolean]('rangeIsEven) + AddressInterpolator invokePrivate rangeIsEven(evenFeature, false) mustBe true + } + } + + property("for rangeIsEven right odd is odd") { + forAll(odd, odd) { (x, y) => + whenever(x != 0 && y != 0) { + val range = Map("RFROMHN" -> x, "RTOHN" -> y) + val oddFeature = new Feature(1, testSchema, range) + val rangeIsEven = PrivateMethod[Boolean]('rangeIsEven) + AddressInterpolator invokePrivate rangeIsEven(oddFeature, false) mustBe false + } + } + } + + property("for calculateAddressRange left even, right odd") { + forAll(odd, odd, even, even, digits) { (r1, r2, l1, l2, a) => + whenever(r1 != 0 && r2 != 0) { + val range = Map("RFROMHN" -> r1, "RTOHN" -> r2, "LFROMHN" -> l1, "LTOHN" -> l2) + val feature = new Feature(1, testSchema, range) + if (a % 2 == 0) calculateAddressRange(feature, a) mustBe (AddressRange(l1, l2)) + else calculateAddressRange(feature, a) mustBe (AddressRange(r1, r2)) + } + } + } + + property("for calculateAddressRange left odd, right even") { + forAll(even, even, odd, odd, digits) { (r1, r2, l1, l2, a) => + whenever(l1 != 0 && l2 != 0) { + val range = Map("RFROMHN" -> r1, "RTOHN" -> r2, "LFROMHN" -> l1, "LTOHN" -> l2) + val feature = new Feature(1, testSchema, range) + if (a % 2 == 0) calculateAddressRange(feature, a) mustBe (AddressRange(r1, r2)) + else calculateAddressRange(feature, a) mustBe (AddressRange(l1, l2)) + } + } + } + +} \ No newline at end of file diff --git a/geocoder/src/test/scala/grasshopper/geocoder/search/census/NumericGenerators.scala b/geocoder/src/test/scala/grasshopper/geocoder/search/census/NumericGenerators.scala index a2b0233..3009e04 100644 --- a/geocoder/src/test/scala/grasshopper/geocoder/search/census/NumericGenerators.scala +++ b/geocoder/src/test/scala/grasshopper/geocoder/search/census/NumericGenerators.scala @@ -16,4 +16,16 @@ trait NumericGenerators { } yield n } + def even: Gen[Int] = { + for { + n <- Gen.choose(-10000, 10000) + } yield n * 2 + } + + def odd: Gen[Int] = { + for { + n <- Gen.choose(-10000, 10000) + } yield n * 2 - 1 + } + } \ No newline at end of file From 96e8ecf3627cce2330a108582fb9bc4562c75815 Mon Sep 17 00:00:00 2001 From: Kenneth Gudel Date: Mon, 2 May 2016 12:08:57 -0400 Subject: [PATCH 9/9] begun writing a test of a complete line, currently failing with return geometry (0,0) regardless of line parameters --- .../search/census/interpolateSpec.scala | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 geocoder/src/test/scala/grasshopper/geocoder/search/census/interpolateSpec.scala diff --git a/geocoder/src/test/scala/grasshopper/geocoder/search/census/interpolateSpec.scala b/geocoder/src/test/scala/grasshopper/geocoder/search/census/interpolateSpec.scala new file mode 100644 index 0000000..402c575 --- /dev/null +++ b/geocoder/src/test/scala/grasshopper/geocoder/search/census/interpolateSpec.scala @@ -0,0 +1,38 @@ +package grasshopper.geocoder.search.census + +import org.scalatest.{ MustMatchers, FlatSpec } +import grasshopper.geocoder.search.census.AddressInterpolator._ +import grasshopper.model.census.AddressRange +import feature._ +import geometry._ + +class InterpolateSpec extends FlatSpec with MustMatchers { + + "A correct point" must "be found on the right side of a horizontal line" in { + val values: Map[String, Any] = Map( + "geometry" -> read("LINESTRING (0 0, 100 0)"), + "FULLNAME" -> "999 Main Street", + "RFROMHN" -> 1, + "RTOHN" -> 101, + "LFROMHN" -> 0, + "LTOHN" -> 100, + "ZIPR" -> "19041", + "ZIPL" -> "19041", + "STATE" -> "CA" + ) + + val addressField = Field("address", StringType()) + val geomField = Field("geometry", GeometryType()) + val numberField = Field("number", IntType()) + val schema = Schema(geomField, addressField, numberField) + val feature = Feature(schema, values) + val addressNumber = 25 + val addressRange = AddressRange(0, 100) + + val newFeature = interpolate(feature, addressRange, addressNumber) + + newFeature.geometry mustBe ((25, .0001)) + + } + +} \ No newline at end of file