From e3664274e33ba586d43fe4c7d9f612a6fecd9d54 Mon Sep 17 00:00:00 2001 From: Arunvel Sriram Date: Sat, 1 Feb 2020 22:35:55 +0530 Subject: [PATCH 1/2] Update .gitignore --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index f04fa9e..f4cd053 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,7 @@ .project dependency-reduced-pom.xml target -*.log \ No newline at end of file +*.log +.DS_Store +.idea +.settings \ No newline at end of file From 28681aa3953e27b8ec97c4ec46f666a435b66ae6 Mon Sep 17 00:00:00 2001 From: Arunvel Sriram Date: Sat, 1 Feb 2020 23:27:02 +0530 Subject: [PATCH 2/2] Add support for writing parsed logs to stdout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - make -o option optional and write to stdout by default - when -o is “-” write output to stdout --- README.txt | 2 ++ .../com/shanyu/hadoop/logparser/LogParser.java | 5 +---- .../com/shanyu/hadoop/logparser/TFileParser.java | 15 ++++++++++++--- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/README.txt b/README.txt index 1cf5849..7b6c82f 100644 --- a/README.txt +++ b/README.txt @@ -17,6 +17,8 @@ For HDInsight clusters, just download your logs on your home container located i Then each of the container logs will be parsed and written to a ".txt" file, along with the original log file. +When -o option is not provided the parsed output will be written to stdout. + [Download and Parse Logs from WASB] You can also use this tool to directly download Yarn application logs from Azure Storage account associated with an HDInsight cluster. diff --git a/src/main/java/com/shanyu/hadoop/logparser/LogParser.java b/src/main/java/com/shanyu/hadoop/logparser/LogParser.java index 497c46d..a654a8e 100644 --- a/src/main/java/com/shanyu/hadoop/logparser/LogParser.java +++ b/src/main/java/com/shanyu/hadoop/logparser/LogParser.java @@ -6,7 +6,6 @@ import java.io.FileWriter; import java.io.FileNotFoundException; import java.io.IOException; -import java.io.File; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; @@ -44,8 +43,7 @@ public static void main( String[] args ) String.format("WASB Account suffix, defaults to '%s'", DEFAULT_WASB_ACCOUNT_SUFFIX))); options.addOption(new Option("i", "applicationId", true, "Application ID")); options.addOption(new Option("d", "localDir", true, "Local Directory Path")); - Option output = new Option("o", "outputDir", true, "Output Directory Name"); - output.setRequired(true); + Option output = new Option("o", "outputDir", true, "Output Directory Name (\"-\" prints to stdout)"); options.addOption(output); CommandLine cmd = null; @@ -114,7 +112,6 @@ else if (!isStrEmpty(wasbAccount) && !isStrEmpty(wasbKey) && !isStrEmpty(wasbCon Path path = new Path(srcPathDir); FileSystem fs = path.getFileSystem(conf); - new File(outputDir).mkdirs(); TFileParser parser = new TFileParser(conf, fs, outputDir); FileStatus[] files = fs.listStatus(path); for(FileStatus file : files) { diff --git a/src/main/java/com/shanyu/hadoop/logparser/TFileParser.java b/src/main/java/com/shanyu/hadoop/logparser/TFileParser.java index 6332986..9861f80 100644 --- a/src/main/java/com/shanyu/hadoop/logparser/TFileParser.java +++ b/src/main/java/com/shanyu/hadoop/logparser/TFileParser.java @@ -8,6 +8,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.File; +import java.io.OutputStream; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; @@ -32,14 +33,22 @@ public TFileParser(Configuration conf, FileSystem fs, String destLocalFolder) { void parseOneFile(Path srcPath) throws FileNotFoundException, IOException { - String destLocalPath = _destLocalFolder + File.separator + srcPath.getName() + ".txt"; System.out.println("==================================="); System.out.println("converting " + srcPath); - System.out.println("writting to " + destLocalPath); + OutputStream outputStream = null; + if(_destLocalFolder == null || _destLocalFolder.equals("") || _destLocalFolder.equals("-")) { + outputStream = System.out; + System.out.println("writting to stdout"); + } else { + new File(_destLocalFolder).mkdirs(); + String destLocalPath = _destLocalFolder + File.separator + srcPath.getName() + ".txt"; + outputStream = new FileOutputStream(destLocalPath); + System.out.println("writting to " + destLocalPath); + } try ( final FSDataInputStream fsdis = _fs.open(srcPath); - final OutputStreamWriter outWriter = new OutputStreamWriter(new FileOutputStream(destLocalPath), "utf-8"); + final OutputStreamWriter outWriter = new OutputStreamWriter(outputStream, "utf-8"); final TFile.Reader reader = new TFile.Reader(fsdis, _fs.getFileStatus(srcPath).getLen(), _conf); final TFile.Reader.Scanner scanner = reader.createScanner()