diff --git a/release/src/main/groovy/TestScripts.groovy b/release/src/main/groovy/TestScripts.groovy index dc2438007ac1..c9d5ab989fe6 100644 --- a/release/src/main/groovy/TestScripts.groovy +++ b/release/src/main/groovy/TestScripts.groovy @@ -37,6 +37,7 @@ class TestScripts { static String bqDataset static String pubsubTopic static String mavenLocalPath + static List backgroundProcesses = Collections.synchronizedList(new ArrayList()) } def TestScripts(String[] args) { @@ -79,6 +80,10 @@ class TestScripts { var.mavenLocalPath = options.mavenLocalPath println "Maven local path: ${var.mavenLocalPath}" } + + Runtime.getRuntime().addShutdownHook(new Thread({ + stopAllBackgroundProcesses() + })) } def ver() { @@ -135,6 +140,37 @@ class TestScripts { } } + // Run a command in the background, returning the Process object. + public Process runBackground(String cmd) { + println cmd + if (cmd.startsWith("mvn ")) { + return _mvnBackground(cmd.substring(4)) + } else { + return _executeBackground(cmd) + } + } + + // Stop/kill a background process and all its descendants. + public void stopProcess(Process proc) { + if (proc != null && proc.isAlive()) { + try { + proc.descendants().forEach { it.destroyForcibly() } + } catch (Throwable ignored) { + } + proc.destroyForcibly() + proc.waitFor(10, java.util.concurrent.TimeUnit.SECONDS) + } + var.backgroundProcesses.remove(proc) + } + + // Stop all active background processes. + public void stopAllBackgroundProcesses() { + def procs = new ArrayList<>(var.backgroundProcesses) + procs.each { proc -> + stopProcess(proc) + } + } + // Check for expected results in actual stdout from previous command, if fails, log errors then exit. public void see(String expected, String actual) { if (!actual.contains(expected)) { @@ -159,6 +195,7 @@ class TestScripts { // Cleanup and print success public void done() { + stopAllBackgroundProcesses() var.startDir.deleteDir() println "[SUCCESS]" System.exit(0) @@ -187,6 +224,26 @@ class TestScripts { return output_text } + // Run a single command asynchronously in the background + private Process _executeBackground(String cmd) { + def shell = "sh -c cmd".split(' ') + shell[2] = cmd + def pb = new ProcessBuilder(shell) + pb.directory(var.curDir) + pb.redirectErrorStream(true) + def proc = pb.start() + var.backgroundProcesses.add(proc) + Thread.startDaemon { + try { + proc.inputStream.eachLine { + println it + } + } catch (Throwable ignored) { + } + } + return proc + } + // Change directory private void _chdir(String subdir) { var.curDir = new File(var.curDir.absolutePath, subdir) @@ -233,8 +290,45 @@ class TestScripts { return _execute(setPath + cmd) } + // Run a maven command in the background + private Process _mvnBackground(String args) { + String mvnlocalPath = var.mavenLocalPath + if (!(var.mavenLocalPath)) { + mvnlocalPath = var.startDir + } + def m2 = new File(mvnlocalPath, ".m2/repository") + m2.mkdirs() + def settings = new File(mvnlocalPath, "settings.xml") + if(!settings.exists()) { + settings.write """ + + ${m2.absolutePath} + + + testrel + + + test.release + ${var.repoUrl} + + + + + + """ + } + def cmd = "mvn ${args} -s ${settings.absolutePath} -Ptestrel -B" + String path = System.getenv("PATH"); + String maven_home = System.getenv("MAVEN_HOME") ?: '/usr/local/maven' + println "Using maven ${maven_home}" + def mvnPath = "${maven_home}/bin" + def setPath = "export PATH=\"${mvnPath}:${path}\" && " + return _executeBackground(setPath + cmd) + } + // Clean up and report error public void error(String text) { + stopAllBackgroundProcesses() var.startDir.deleteDir() println "[ERROR] $text" System.exit(1) diff --git a/release/src/main/groovy/mobilegaming-java-dataflow.groovy b/release/src/main/groovy/mobilegaming-java-dataflow.groovy index 51ea528a7638..96cd557562ce 100644 --- a/release/src/main/groovy/mobilegaming-java-dataflow.groovy +++ b/release/src/main/groovy/mobilegaming-java-dataflow.groovy @@ -138,19 +138,13 @@ class LeaderBoardRunner { } println "Tables ${userTable} and ${teamTable} created successfully." - def InjectorThread = Thread.start() { - t.run(mobileGamingCommands.createInjectorCommand()) - } + def injectorProcess = t.runBackground(mobileGamingCommands.createInjectorCommand()) String jobName = "leaderboard-validation-" + new Date().getTime() + "-" + new Random().nextInt(1000) - def LeaderBoardThread = Thread.start() { - if (useStreamingEngine) { - t.run(mobileGamingCommands.createPipelineCommand( - "LeaderBoardWithStreamingEngine", runner, jobName, "LeaderBoard")) - } else { - t.run(mobileGamingCommands.createPipelineCommand("LeaderBoard", runner, jobName)) - } - } + def leaderBoardProcess = useStreamingEngine ? + t.runBackground(mobileGamingCommands.createPipelineCommand( + "LeaderBoardWithStreamingEngine", runner, jobName, "LeaderBoard")) : + t.runBackground(mobileGamingCommands.createPipelineCommand("LeaderBoard", runner, jobName)) t.run("gcloud dataflow jobs list | grep pyflow-wordstream-candidate | grep Running | cut -d' ' -f1") @@ -175,8 +169,8 @@ class LeaderBoardRunner { println "Waiting for pipeline to produce more results..." sleep(60000) // wait for 1 min } - InjectorThread.stop() - LeaderBoardThread.stop() + t.stopProcess(injectorProcess) + t.stopProcess(leaderBoardProcess) t.run("""RUNNING_JOB=`gcloud dataflow jobs list | grep ${jobName} | grep Running | cut -d' ' -f1` if [ ! -z "\${RUNNING_JOB}" ] then diff --git a/release/src/main/groovy/mobilegaming-java-direct.groovy b/release/src/main/groovy/mobilegaming-java-direct.groovy index 34eab4c00768..3985c7e59634 100644 --- a/release/src/main/groovy/mobilegaming-java-direct.groovy +++ b/release/src/main/groovy/mobilegaming-java-direct.groovy @@ -98,14 +98,10 @@ while (!tables.contains(userTable) || !tables.contains(teamTable)) { } println "Tables ${userTable} and ${teamTable} created successfully." -def InjectorThread = Thread.start() { - t.run(mobileGamingCommands.createInjectorCommand()) -} +def injectorProcess = t.runBackground(mobileGamingCommands.createInjectorCommand()) jobName = "leaderboard-validation-" + new Date().getTime() + "-" + new Random().nextInt(1000) -def LeaderBoardThread = Thread.start() { - t.run(mobileGamingCommands.createPipelineCommand("LeaderBoard", runner, jobName)) -} +def leaderBoardProcess = t.runBackground(mobileGamingCommands.createPipelineCommand("LeaderBoard", runner, jobName)) // verify outputs in BQ tables def startTime = System.currentTimeMillis() @@ -128,8 +124,8 @@ while ((System.currentTimeMillis() - startTime)/60000 < mobileGamingCommands.EXE println "Waiting for pipeline to produce more results..." sleep(60000) // wait for 1 min } -InjectorThread.stop() -LeaderBoardThread.stop() +t.stopProcess(injectorProcess) +t.stopProcess(leaderBoardProcess) if(!isSuccess){ t.error("FAILED: Failed running LeaderBoard on DirectRunner")