From d6bab3c921fee93f8f27204a02d83c266f5c7f19 Mon Sep 17 00:00:00 2001 From: Miguel Angel Fernandez Diaz Date: Tue, 3 May 2016 14:56:55 +0200 Subject: [PATCH 01/13] [CROSSDATA] [CONFIG] Pom files with parent and artifactId to fixed Scala version --- cassandra/pom.xml | 4 ++-- common/pom.xml | 2 +- core/pom.xml | 4 ++-- dist/pom.xml | 4 ++-- driver/pom.xml | 4 ++-- elasticsearch/pom.xml | 4 ++-- examples/pom.xml | 4 ++-- mongodb/pom.xml | 4 ++-- server/pom.xml | 4 ++-- streaming/pom.xml | 4 ++-- testsIT/pom.xml | 4 ++-- 11 files changed, 21 insertions(+), 21 deletions(-) diff --git a/cassandra/pom.xml b/cassandra/pom.xml index 77628b05f..684af0a9f 100644 --- a/cassandra/pom.xml +++ b/cassandra/pom.xml @@ -21,11 +21,11 @@ 4.0.0 com.stratio.crossdata - parent_${scala.binary.version} + parent_2.10 1.3.0-SNAPSHOT com.stratio.crossdata.connector - crossdata-cassandra_${scala.binary.version} + crossdata-cassandra_2.10 jar Cassandra Connector Crossdata Cassandra Connector diff --git a/common/pom.xml b/common/pom.xml index 525b4b9ac..44aa04d79 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -21,7 +21,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - parent_${scala.binary.version} + parent_2.10 com.stratio.crossdata 1.3.0-SNAPSHOT diff --git a/core/pom.xml b/core/pom.xml index a798c2b08..11ab0e056 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -21,11 +21,11 @@ 4.0.0 com.stratio.crossdata - parent_${scala.binary.version} + parent_2.10 1.3.0-SNAPSHOT - crossdata-core_${scala.binary.version} + crossdata-core_2.10 Core jar Some improvements for SparkSQL diff --git a/dist/pom.xml b/dist/pom.xml index 266d8f4cc..3844a72eb 100644 --- a/dist/pom.xml +++ b/dist/pom.xml @@ -6,11 +6,11 @@ com.stratio.crossdata - parent_${scala.binary.version} + parent_2.10 1.3.0-SNAPSHOT - dist_${scala.binary.version} + dist_2.10 1.3.0-SNAPSHOT diff --git a/driver/pom.xml b/driver/pom.xml index 37cfaa051..727f95acc 100644 --- a/driver/pom.xml +++ b/driver/pom.xml @@ -21,12 +21,12 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - parent_${scala.binary.version} + parent_2.10 com.stratio.crossdata 1.3.0-SNAPSHOT - crossdata-driver_${scala.binary.version} + crossdata-driver_2.10 Driver Driver for Crossdata in order to communicate with the Crossdata Servers jar diff --git a/elasticsearch/pom.xml b/elasticsearch/pom.xml index dd44d42a7..5bb881056 100644 --- a/elasticsearch/pom.xml +++ b/elasticsearch/pom.xml @@ -22,11 +22,11 @@ 4.0.0 com.stratio.crossdata - parent_${scala.binary.version} + parent_2.10 1.3.0-SNAPSHOT com.stratio.crossdata.connector - crossdata-elasticsearch_${scala.binary.version} + crossdata-elasticsearch_2.10 jar Elasticsearch Connector Crossdata Elasticsearch Connector diff --git a/examples/pom.xml b/examples/pom.xml index 09f55d429..e6c592cb9 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -21,11 +21,11 @@ 4.0.0 com.stratio - parent_${scala.binary.version} + parent_2.10 0.5.0 - crossdata-examples_${scala.binary.version} + crossdata-examples_2.10 1.3.0-SNAPSHOT jar Examples diff --git a/mongodb/pom.xml b/mongodb/pom.xml index 98b6a5e3f..149250dfb 100644 --- a/mongodb/pom.xml +++ b/mongodb/pom.xml @@ -21,11 +21,11 @@ 4.0.0 com.stratio.crossdata - parent_${scala.binary.version} + parent_2.10 1.3.0-SNAPSHOT com.stratio.crossdata.connector - crossdata-mongodb_${scala.binary.version} + crossdata-mongodb_2.10 jar MongoDB Connector Crossdata MongoDB connector diff --git a/server/pom.xml b/server/pom.xml index f9a91c9e1..b3f7ebe5b 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -21,12 +21,12 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - parent_${scala.binary.version} + parent_2.10 com.stratio.crossdata 1.3.0-SNAPSHOT - crossdata-server_${scala.binary.version} + crossdata-server_2.10 jar Server Crossdata Server diff --git a/streaming/pom.xml b/streaming/pom.xml index f08cd051f..c61c18e15 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -3,13 +3,13 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - parent_${scala.binary.version} + parent_2.10 com.stratio.crossdata 1.3.0-SNAPSHOT 4.0.0 - crossdata-streaming_${scala.binary.version} + crossdata-streaming_2.10 jar Streaming Crossdata Streaming diff --git a/testsIT/pom.xml b/testsIT/pom.xml index 60d5c3d36..80e81925c 100644 --- a/testsIT/pom.xml +++ b/testsIT/pom.xml @@ -21,12 +21,12 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - parent_${scala.binary.version} + parent_2.10 com.stratio.crossdata 1.3.0-SNAPSHOT - crossdata-testsIT_${scala.binary.version} + crossdata-testsIT_2.10 TestsIT Integration tests for Crossdata jar From 666cbab96ab2ceb83b5e5e9af4a65428dfc53c86 Mon Sep 17 00:00:00 2001 From: Miguel Angel Fernandez Diaz Date: Fri, 9 Sep 2016 16:27:49 +0200 Subject: [PATCH 02/13] [CROSSDATA-577] Service discovery: test skeleton --- .../crossdata/server/ServiceDiscoveryIT.scala | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala diff --git a/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala b/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala new file mode 100644 index 000000000..4f74e0ceb --- /dev/null +++ b/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2015 Stratio (http://stratio.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.stratio.crossdata.server + +import com.stratio.crossdata.test.BaseXDTest +import com.typesafe.config.{ConfigFactory, ConfigValueFactory} +import org.junit.runner.RunWith +import org.scalatest.BeforeAndAfterAll +import org.scalatest.junit.JUnitRunner + +@RunWith(classOf[JUnitRunner]) +class ServiceDiscoveryIT extends BaseXDTest with BeforeAndAfterAll { + + override def beforeAll(): Unit = super.beforeAll() + + override def afterAll(): Unit = super.afterAll() + + "A Crossdata Server" should "write its hostname:port in ZK when service discovery is activated" in { + + val TestHost = "127.0.0.1" + val AkkaPort = 13456 + val HzPort = 5789 + + val testConfig = ConfigFactory.empty + .withValue("akka.remote.netty.tcp.hostname", ConfigValueFactory.fromAnyRef(TestHost)) + .withValue("akka.remote.netty.tcp.port", ConfigValueFactory.fromAnyRef(AkkaPort)) + .withValue("service-discovery.activated", ConfigValueFactory.fromAnyRef(true)) + + val testServer = new CrossdataServer(Some(testConfig)) + + testServer.start + } + +} From 15e4f6e8e89bf2ee493a3ee44914723e153f2605 Mon Sep 17 00:00:00 2001 From: Miguel Angel Fernandez Diaz Date: Sun, 11 Sep 2016 11:57:19 +0200 Subject: [PATCH 03/13] [CROSSDATA-577] Service discovery: Adding integration test --- .../crossdata/server/CrossdataServer.scala | 17 +++++++-- .../crossdata/server/ServiceDiscoveryIT.scala | 36 ++++++++++++++----- 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/server/src/main/scala/com/stratio/crossdata/server/CrossdataServer.scala b/server/src/main/scala/com/stratio/crossdata/server/CrossdataServer.scala index bb262d44c..803604c1a 100644 --- a/server/src/main/scala/com/stratio/crossdata/server/CrossdataServer.scala +++ b/server/src/main/scala/com/stratio/crossdata/server/CrossdataServer.scala @@ -45,7 +45,7 @@ import scala.concurrent.{ExecutionContext, Future} import scala.util.{Failure, Try} -class CrossdataServer(progrConfig: Option[Config] = None) extends ServerConfig { +class CrossdataServer(progrConfig: Option[Config] = None, hzMembers: Option[Set[String]] = None) extends ServerConfig { override lazy val logger = Logger.getLogger(classOf[CrossdataServer]) @@ -55,7 +55,20 @@ class CrossdataServer(progrConfig: Option[Config] = None) extends ServerConfig { private val serverConfig = progrConfig map (_.withFallback(config)) getOrElse (config) - private val hzConfig: HzConfig = new XmlConfigBuilder().build() + private val hzConfig: HzConfig = { + val baseConfig = new XmlConfigBuilder().build() + hzMembers map { members => + insertHzMembers(baseConfig, members) + } getOrElse { + baseConfig + } + } + + private def insertHzMembers(hConfig: HzConfig, members: Set[String]) = + hConfig.setNetworkConfig( + hConfig.getNetworkConfig.setJoin( + hConfig.getNetworkConfig.getJoin.setTcpIpConfig( + hConfig.getNetworkConfig.getJoin.getTcpIpConfig.setMembers(members.toList)))) private def startDiscoveryClient(sdConfig: SDCH): CuratorFramework = { diff --git a/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala b/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala index 4f74e0ceb..5edda7485 100644 --- a/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala +++ b/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala @@ -17,6 +17,7 @@ package com.stratio.crossdata.server import com.stratio.crossdata.test.BaseXDTest import com.typesafe.config.{ConfigFactory, ConfigValueFactory} +import org.apache.curator.CuratorZookeeperClient import org.junit.runner.RunWith import org.scalatest.BeforeAndAfterAll import org.scalatest.junit.JUnitRunner @@ -24,24 +25,41 @@ import org.scalatest.junit.JUnitRunner @RunWith(classOf[JUnitRunner]) class ServiceDiscoveryIT extends BaseXDTest with BeforeAndAfterAll { - override def beforeAll(): Unit = super.beforeAll() + import ServiceDiscoveryConstants._ - override def afterAll(): Unit = super.afterAll() - - "A Crossdata Server" should "write its hostname:port in ZK when service discovery is activated" in { - - val TestHost = "127.0.0.1" - val AkkaPort = 13456 - val HzPort = 5789 + var testServer: CrossdataServer = _ + override def beforeAll(): Unit = { val testConfig = ConfigFactory.empty .withValue("akka.remote.netty.tcp.hostname", ConfigValueFactory.fromAnyRef(TestHost)) .withValue("akka.remote.netty.tcp.port", ConfigValueFactory.fromAnyRef(AkkaPort)) .withValue("service-discovery.activated", ConfigValueFactory.fromAnyRef(true)) - val testServer = new CrossdataServer(Some(testConfig)) + testServer = new CrossdataServer(Some(testConfig), Some(Set(s"$TestHost:$HzPort"))) testServer.start + + + + } + + override def afterAll(): Unit = { + testServer.stop + } + + "A Crossdata Server" should "write its hostname:port in ZK when service discovery is activated" in { + + new CuratorZookeeperClient() + } } + +object ServiceDiscoveryConstants { + + val TestHost = "127.0.0.1" + val AkkaPort = 13456 + val HzPort = 5789 + + +} From 3b2a2897ae116104b252e4e8eb59f560e932a240 Mon Sep 17 00:00:00 2001 From: Miguel Angel Fernandez Diaz Date: Sun, 11 Sep 2016 16:56:18 +0200 Subject: [PATCH 04/13] [CROSSDATA-577] Service discovery: Integration test --- .../crossdata/server/ServiceDiscoveryIT.scala | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala b/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala index 5edda7485..072b98d76 100644 --- a/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala +++ b/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala @@ -15,9 +15,12 @@ */ package com.stratio.crossdata.server +import com.stratio.crossdata.server.discovery.ServiceDiscoveryConfigHelper import com.stratio.crossdata.test.BaseXDTest import com.typesafe.config.{ConfigFactory, ConfigValueFactory} import org.apache.curator.CuratorZookeeperClient +import org.apache.curator.framework.CuratorFrameworkFactory +import org.apache.curator.retry.ExponentialBackoffRetry import org.junit.runner.RunWith import org.scalatest.BeforeAndAfterAll import org.scalatest.junit.JUnitRunner @@ -27,6 +30,8 @@ class ServiceDiscoveryIT extends BaseXDTest with BeforeAndAfterAll { import ServiceDiscoveryConstants._ + val ZkConnectionString = sys.env("XD_ZOOKEEPER_CONNECTION_STRING") + var testServer: CrossdataServer = _ override def beforeAll(): Unit = { @@ -38,9 +43,6 @@ class ServiceDiscoveryIT extends BaseXDTest with BeforeAndAfterAll { testServer = new CrossdataServer(Some(testConfig), Some(Set(s"$TestHost:$HzPort"))) testServer.start - - - } override def afterAll(): Unit = { @@ -48,18 +50,23 @@ class ServiceDiscoveryIT extends BaseXDTest with BeforeAndAfterAll { } "A Crossdata Server" should "write its hostname:port in ZK when service discovery is activated" in { + val curatorClient = CuratorFrameworkFactory.newClient( + ZkConnectionString, + new ExponentialBackoffRetry(1000, 3)) + curatorClient.blockUntilConnected + val currentSeeds = new String(curatorClient.getData.forPath(ServiceDiscoveryConfigHelper.DefaultSeedsPath)) - new CuratorZookeeperClient() + currentSeeds should be s"$TestHost:$AkkaPort" + val currentMembers = new String(curatorClient.getData.forPath(ServiceDiscoveryConfigHelper.DefaultProviderPath)) + + currentMembers should be s"$TestHost:$HzPort" } } object ServiceDiscoveryConstants { - val TestHost = "127.0.0.1" val AkkaPort = 13456 val HzPort = 5789 - - } From 6e7eee975577b161f0d271794d203404e3baf152 Mon Sep 17 00:00:00 2001 From: Miguel Angel Fernandez Diaz Date: Sun, 11 Sep 2016 17:10:16 +0200 Subject: [PATCH 05/13] [CROSSDATA-577] Service discovery: Problem in test fixed --- .../com/stratio/crossdata/server/ServiceDiscoveryIT.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala b/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala index 072b98d76..87ea2c138 100644 --- a/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala +++ b/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala @@ -56,11 +56,11 @@ class ServiceDiscoveryIT extends BaseXDTest with BeforeAndAfterAll { curatorClient.blockUntilConnected val currentSeeds = new String(curatorClient.getData.forPath(ServiceDiscoveryConfigHelper.DefaultSeedsPath)) - currentSeeds should be s"$TestHost:$AkkaPort" + currentSeeds should be (s"$TestHost:$AkkaPort") val currentMembers = new String(curatorClient.getData.forPath(ServiceDiscoveryConfigHelper.DefaultProviderPath)) - currentMembers should be s"$TestHost:$HzPort" + currentMembers should be (s"$TestHost:$HzPort") } } From d21ee2d6e72cb344bddfcb763e91979c93f75713 Mon Sep 17 00:00:00 2001 From: Miguel Angel Fernandez Diaz Date: Sun, 11 Sep 2016 18:04:21 +0200 Subject: [PATCH 06/13] [CROSSDATA-577] Service discovery: Improvements in test --- .../crossdata/server/ServiceDiscoveryIT.scala | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala b/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala index 87ea2c138..f13af9292 100644 --- a/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala +++ b/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala @@ -18,7 +18,6 @@ package com.stratio.crossdata.server import com.stratio.crossdata.server.discovery.ServiceDiscoveryConfigHelper import com.stratio.crossdata.test.BaseXDTest import com.typesafe.config.{ConfigFactory, ConfigValueFactory} -import org.apache.curator.CuratorZookeeperClient import org.apache.curator.framework.CuratorFrameworkFactory import org.apache.curator.retry.ExponentialBackoffRetry import org.junit.runner.RunWith @@ -30,15 +29,20 @@ class ServiceDiscoveryIT extends BaseXDTest with BeforeAndAfterAll { import ServiceDiscoveryConstants._ - val ZkConnectionString = sys.env("XD_ZOOKEEPER_CONNECTION_STRING") + val ZkConnectionString = sys.env.get("XD_ZOOKEEPER_CONNECTION_STRING") + var testServer: CrossdataServer = _ override def beforeAll(): Unit = { + val testConfig = ConfigFactory.empty .withValue("akka.remote.netty.tcp.hostname", ConfigValueFactory.fromAnyRef(TestHost)) .withValue("akka.remote.netty.tcp.port", ConfigValueFactory.fromAnyRef(AkkaPort)) .withValue("service-discovery.activated", ConfigValueFactory.fromAnyRef(true)) + .withValue( + "config.spark.jars", + ConfigValueFactory.fromAnyRef(s"server/target/2.11/crossdata-server-jar-with-dependencies.jar")) testServer = new CrossdataServer(Some(testConfig), Some(Set(s"$TestHost:$HzPort"))) @@ -50,8 +54,11 @@ class ServiceDiscoveryIT extends BaseXDTest with BeforeAndAfterAll { } "A Crossdata Server" should "write its hostname:port in ZK when service discovery is activated" in { + + ZkConnectionString.isDefined should be (true) + val curatorClient = CuratorFrameworkFactory.newClient( - ZkConnectionString, + ZkConnectionString.get, new ExponentialBackoffRetry(1000, 3)) curatorClient.blockUntilConnected val currentSeeds = new String(curatorClient.getData.forPath(ServiceDiscoveryConfigHelper.DefaultSeedsPath)) From f2c07839b973c65365a38a728be901e998384fc9 Mon Sep 17 00:00:00 2001 From: Miguel Angel Fernandez Diaz Date: Sun, 11 Sep 2016 23:25:55 +0200 Subject: [PATCH 07/13] [CROSSDATA-577] Service discovery: Avoid endless beforeAll --- .../com/stratio/crossdata/server/ServiceDiscoveryIT.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala b/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala index f13af9292..b2828021b 100644 --- a/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala +++ b/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala @@ -24,6 +24,9 @@ import org.junit.runner.RunWith import org.scalatest.BeforeAndAfterAll import org.scalatest.junit.JUnitRunner +import scala.concurrent.{Await, Future} +import scala.concurrent.duration._ + @RunWith(classOf[JUnitRunner]) class ServiceDiscoveryIT extends BaseXDTest with BeforeAndAfterAll { @@ -44,7 +47,7 @@ class ServiceDiscoveryIT extends BaseXDTest with BeforeAndAfterAll { "config.spark.jars", ConfigValueFactory.fromAnyRef(s"server/target/2.11/crossdata-server-jar-with-dependencies.jar")) - testServer = new CrossdataServer(Some(testConfig), Some(Set(s"$TestHost:$HzPort"))) + testServer = Await.result(Future(new CrossdataServer(Some(testConfig), Some(Set(s"$TestHost:$HzPort")))), 2 minutes) testServer.start } From 45d0784e5c5b40e453eb66f15d6464e2fc29902c Mon Sep 17 00:00:00 2001 From: Miguel Angel Fernandez Diaz Date: Fri, 16 Sep 2016 16:15:29 +0200 Subject: [PATCH 08/13] [CROSSDATA-577] Service discovery: Integration test - Compilation error fixed --- .../scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala b/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala index b2828021b..5f759e4d9 100644 --- a/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala +++ b/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala @@ -47,6 +47,7 @@ class ServiceDiscoveryIT extends BaseXDTest with BeforeAndAfterAll { "config.spark.jars", ConfigValueFactory.fromAnyRef(s"server/target/2.11/crossdata-server-jar-with-dependencies.jar")) + import scala.concurrent.ExecutionContext.Implicits.global testServer = Await.result(Future(new CrossdataServer(Some(testConfig), Some(Set(s"$TestHost:$HzPort")))), 2 minutes) testServer.start From 80282cffc17b23aeb5680967ba483d075acb43a2 Mon Sep 17 00:00:00 2001 From: Miguel Angel Fernandez Diaz Date: Sat, 17 Sep 2016 16:14:14 +0200 Subject: [PATCH 09/13] [CROSSDATA] Crossdata versions up-to-date --- README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0ca062436..be830e938 100644 --- a/README.md +++ b/README.md @@ -46,10 +46,13 @@ Spark Compatibility | Crossdata Version | Spark Version | |-------------------|:--------------| +| 1.6.X | 1.6.X | +| 1.5.X | 1.6.X | +| 1.4.X | 1.6.X | | 1.3.X | 1.6.X | -| 1.2.X | 1.5.X | -| 1.1.X | 1.5.X | -| 1.0.X | 1.5.X | +| 1.2.X | 1.5.X | +| 1.1.X | 1.5.X | +| 1.0.X | 1.5.X | ============= From 3b626c4c5ab092a8555545e0d69108c2fcd73f9e Mon Sep 17 00:00:00 2001 From: Miguel Angel Fernandez Diaz Date: Sat, 17 Sep 2016 16:49:13 +0200 Subject: [PATCH 10/13] [CROSSDATA-577] Service discovery: Integration test - Avoiding endless connection --- .../com/stratio/crossdata/server/ServiceDiscoveryIT.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala b/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala index 5f759e4d9..13be762f3 100644 --- a/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala +++ b/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala @@ -50,11 +50,11 @@ class ServiceDiscoveryIT extends BaseXDTest with BeforeAndAfterAll { import scala.concurrent.ExecutionContext.Implicits.global testServer = Await.result(Future(new CrossdataServer(Some(testConfig), Some(Set(s"$TestHost:$HzPort")))), 2 minutes) - testServer.start + Await.result(Future(testServer.start), 2 minutes) } override def afterAll(): Unit = { - testServer.stop + Await.result(Future(testServer.stop), 2 minutes) } "A Crossdata Server" should "write its hostname:port in ZK when service discovery is activated" in { From b29e5df9115cfbcb29f30267e4f8d7f2b720bab4 Mon Sep 17 00:00:00 2001 From: Miguel Angel Fernandez Diaz Date: Mon, 19 Sep 2016 12:09:20 +0200 Subject: [PATCH 11/13] [CROSSDATA-577] Service discovery: Integration test - Using CD ZK --- .../stratio/crossdata/server/ServiceDiscoveryIT.scala | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala b/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala index 13be762f3..5129d0463 100644 --- a/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala +++ b/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala @@ -26,14 +26,16 @@ import org.scalatest.junit.JUnitRunner import scala.concurrent.{Await, Future} import scala.concurrent.duration._ +import scala.util.Try @RunWith(classOf[JUnitRunner]) class ServiceDiscoveryIT extends BaseXDTest with BeforeAndAfterAll { import ServiceDiscoveryConstants._ - val ZkConnectionString = sys.env.get("XD_ZOOKEEPER_CONNECTION_STRING") - + val ZookeeperStreamingConnectionKey = "streaming.catalog.zookeeper.connectionString" + val ZookeeperConnection: Option[String] = + Try(ConfigFactory.load().getString(ZookeeperStreamingConnectionKey)).toOption var testServer: CrossdataServer = _ @@ -59,10 +61,10 @@ class ServiceDiscoveryIT extends BaseXDTest with BeforeAndAfterAll { "A Crossdata Server" should "write its hostname:port in ZK when service discovery is activated" in { - ZkConnectionString.isDefined should be (true) + ZookeeperConnection.isDefined should be (true) val curatorClient = CuratorFrameworkFactory.newClient( - ZkConnectionString.get, + ZookeeperConnection.get, new ExponentialBackoffRetry(1000, 3)) curatorClient.blockUntilConnected val currentSeeds = new String(curatorClient.getData.forPath(ServiceDiscoveryConfigHelper.DefaultSeedsPath)) From 452a8241245707e7bc5ab0b657e0b9ca331d640f Mon Sep 17 00:00:00 2001 From: Miguel Angel Fernandez Diaz Date: Mon, 19 Sep 2016 12:30:13 +0200 Subject: [PATCH 12/13] [CROSSDATA-577] Service discovery: Integration test - Execution Context error fixed --- .../scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala b/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala index 5129d0463..9f3fec824 100644 --- a/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala +++ b/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala @@ -56,6 +56,7 @@ class ServiceDiscoveryIT extends BaseXDTest with BeforeAndAfterAll { } override def afterAll(): Unit = { + import scala.concurrent.ExecutionContext.Implicits.global Await.result(Future(testServer.stop), 2 minutes) } From 540100437a0b0271d7a0116dd1918bb4ea2055a1 Mon Sep 17 00:00:00 2001 From: Miguel Angel Fernandez Diaz Date: Mon, 19 Sep 2016 12:36:30 +0200 Subject: [PATCH 13/13] [CROSSDATA-577] Service discovery: Integration test - Execution Context error fixed --- .../com/stratio/crossdata/server/ServiceDiscoveryIT.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala b/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala index 9f3fec824..83cfa8ff5 100644 --- a/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala +++ b/server/src/test/scala/com/stratio/crossdata/server/ServiceDiscoveryIT.scala @@ -15,6 +15,8 @@ */ package com.stratio.crossdata.server +import java.util.concurrent.Executors + import com.stratio.crossdata.server.discovery.ServiceDiscoveryConfigHelper import com.stratio.crossdata.test.BaseXDTest import com.typesafe.config.{ConfigFactory, ConfigValueFactory} @@ -24,7 +26,7 @@ import org.junit.runner.RunWith import org.scalatest.BeforeAndAfterAll import org.scalatest.junit.JUnitRunner -import scala.concurrent.{Await, Future} +import scala.concurrent.{Await, ExecutionContext, Future} import scala.concurrent.duration._ import scala.util.Try @@ -39,6 +41,8 @@ class ServiceDiscoveryIT extends BaseXDTest with BeforeAndAfterAll { var testServer: CrossdataServer = _ + implicit val executionContext = ExecutionContext.fromExecutor(Executors.newFixedThreadPool(8)) + override def beforeAll(): Unit = { val testConfig = ConfigFactory.empty @@ -49,14 +53,12 @@ class ServiceDiscoveryIT extends BaseXDTest with BeforeAndAfterAll { "config.spark.jars", ConfigValueFactory.fromAnyRef(s"server/target/2.11/crossdata-server-jar-with-dependencies.jar")) - import scala.concurrent.ExecutionContext.Implicits.global testServer = Await.result(Future(new CrossdataServer(Some(testConfig), Some(Set(s"$TestHost:$HzPort")))), 2 minutes) Await.result(Future(testServer.start), 2 minutes) } override def afterAll(): Unit = { - import scala.concurrent.ExecutionContext.Implicits.global Await.result(Future(testServer.stop), 2 minutes) }