diff --git a/bin/hipo-diffBanks b/bin/hipo-diffBanks new file mode 100755 index 0000000000..af938f6d13 --- /dev/null +++ b/bin/hipo-diffBanks @@ -0,0 +1,10 @@ +#!/bin/bash + +. `dirname $0`/../libexec/env.sh + +export MALLOC_ARENA_MAX=1 + +java ${JAVA_OPTS-} -Xmx1536m -Xms1024m -XX:+UseSerialGC \ + -cp ${COATJAVA_CLASSPATH:-''} \ + org.jlab.utils.HipoDiffBanks \ + $* diff --git a/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java b/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java index 52f82c4e2e..8a06d076bd 100644 --- a/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java +++ b/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java @@ -7,6 +7,7 @@ import org.jlab.jnp.hipo4.data.Event; import org.jlab.jnp.hipo4.data.Schema; import org.jlab.jnp.hipo4.data.SchemaFactory; +import org.jlab.jnp.hipo4.io.HipoWriter; import org.jlab.utils.options.OptionParser; public class HipoDiff { @@ -14,8 +15,8 @@ public class HipoDiff { /** * Bank sortable by any integer columns. */ - static class SortedBank extends Bank { - SortedBank(Schema s) { super(s); } + static class DiffBank extends Bank { + DiffBank(Schema s) { super(s); } /** * @param index the bank column indices to sort on * @return the sorted row indices @@ -42,6 +43,43 @@ int[] getSorted(int... index) { } return rows; } + /** + * @param b the bank to compare with + * @return the resulting diff bank + */ + public Bank getDiff(Bank b) { + Bank diff = new Bank(getSchema()); + int rows = Math.min(getRows(), b.getRows()); + int ncols = getSchema().getElements(); + for (int row = 0; row < rows; row++) { + for (int col = 0; col < ncols; col++) { + switch (getSchema().getType(col)) { + case 1: // byte + diff.putByte(col, row, (byte)(b.getByte(col,row) - getByte(col,row))); + break; + case 2: // short + diff.putShort(col, row, (short)(b.getShort(col,row) - getShort(col,row))); + break; + case 3: // int + diff.putInt(col, row, b.getInt(col,row) - getInt(col,row)); + break; + case 4: // float + diff.putFloat(col, row, b.getFloat(col,row) - getFloat(col,row)); + break; + case 5: // double + diff.putDouble(col, row, b.getDouble(col,row) - getDouble(col,row)); + break; + case 6: // long + diff.putLong(col, row, b.getLong(col,row) - getLong(col,row)); + break; + default: + // unhandled type (arrays, etc.) + break; + } + } + } + return diff; + } } static int nrow = 0; @@ -57,9 +95,11 @@ int[] getSorted(int... index) { static boolean verboseMode; static boolean quietMode; static Bank runConfigBank; + static HipoWriter writer; + static Event event; - static ArrayList banksA = new ArrayList<>(); - static ArrayList banksB = new ArrayList<>(); + static ArrayList banksA = new ArrayList<>(); + static ArrayList banksB = new ArrayList<>(); static HashMap> badEntries = new HashMap<>(); public static void main(String args[]) { @@ -70,6 +110,7 @@ public static void main(String args[]) { op.addOption("-Q", null, "verbose mode"); op.addOption("-b", null, "name of bank to diff"); op.addOption("-s", null, "sort on column index"); + op.addOption("-o", null, "output HIPO diff file"); op.setRequiresInputList(true); op.parse(args); if (op.getInputList().size() != 2) { @@ -96,18 +137,27 @@ public static void main(String args[]) { if (op.getOption("-b").stringValue() == null) { for (Schema s : sf.getSchemaList()) { - banksA.add(new SortedBank(s)); - banksB.add(new SortedBank(s)); + banksA.add(new DiffBank(s)); + banksB.add(new DiffBank(s)); } } else { - banksA.add(new SortedBank(sf.getSchema(op.getOption("-b").stringValue()))); - banksB.add(new SortedBank(sf.getSchema(op.getOption("-b").stringValue()))); + banksA.add(new DiffBank(sf.getSchema(op.getOption("-b").stringValue()))); + banksB.add(new DiffBank(sf.getSchema(op.getOption("-b").stringValue()))); + } + + if (op.getOption("-o").stringValue() != null) { + writer = new HipoWriter(); + writer.getSchemaFactory().copy(readerA.getSchemaFactory()); + writer.open(op.getOption("-o").stringValue()); } compare(readerA, readerB); + + if (writer != null) writer.close(); } - public static void compare(HipoReader a, HipoReader b) { + public static int compare(HipoReader a, HipoReader b) { + int ret=0; Event eventA = new Event(); Event eventB = new Event(); while (a.hasNext() && b.hasNext() && (nmax < 1 || nevent < nmax)) { @@ -115,7 +165,7 @@ public static void compare(HipoReader a, HipoReader b) { a.nextEvent(eventA); b.nextEvent(eventB); eventA.read(runConfigBank); - compare(eventA, eventB); + ret += compare(eventA, eventB); } System.out.println("\n Analyzed " + nevent + " with " + nbadevent + " bad banks"); System.out.println(nbadrow + "/" + nrow + " mismatched rows"); @@ -124,17 +174,26 @@ public static void compare(HipoReader a, HipoReader b) { System.out.println(name + " " + badEntries.get(name)); } System.exit(nbadevent + nbadrow + nbadentry); + return ret; } - public static void compare(Event a, Event b) { + public static int compare(Event a, Event b) { + int ret = 0; for (int i = 0; i < banksA.size(); i++) { a.read(banksA.get(i)); b.read(banksB.get(i)); - compare(banksA.get(i), banksB.get(i)); + if (writer != null) { + event.reset(); + event.write(banksA.get(i).getDiff(banksB.get(i))); + writer.addEvent(event); + } + ret += compare(banksA.get(i), banksB.get(i)); } + return ret; } - public static void compare(SortedBank a, SortedBank b) { + public static int compare(DiffBank a, DiffBank b) { + int ret=0; if (a.getRows() != b.getRows()) { System.out.println("========================= Different number of rows:"); runConfigBank.show(); @@ -142,7 +201,7 @@ public static void compare(SortedBank a, SortedBank b) { b.show(); nbadevent++; System.out.println("========================="); - return; + return ++ret; } int[] rowsA = sortIndex == null ? null : a.getSorted(sortIndex); int[] rowsB = sortIndex == null ? null : b.getSorted(sortIndex); @@ -207,11 +266,13 @@ public static void compare(SortedBank a, SortedBank b) { m.put(elementName, 0); } m.put(elementName, m.get(elementName) + 1); + ret++; } } if (mismatch) { nbadrow++; } } + return 0; } } diff --git a/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiffBanks.java b/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiffBanks.java new file mode 100644 index 0000000000..2e24eb63b7 --- /dev/null +++ b/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiffBanks.java @@ -0,0 +1,167 @@ +package org.jlab.utils; + +import org.jlab.jnp.hipo4.io.HipoReader; +import org.jlab.jnp.hipo4.io.HipoWriter; +import org.jlab.jnp.hipo4.data.Event; +import org.jlab.jnp.hipo4.data.Bank; +import org.jlab.jnp.hipo4.data.Schema; +import org.jlab.jnp.hipo4.data.SchemaFactory; + +public class HipoDiffBanks { + + public static void main(String[] args) throws Exception { + if (args.length < 3) { + System.err.println("usage: HipoDiffBanks run1.hipo run2.hipo out.hipo"); + System.exit(1); + } + + String f1 = args[0]; + String f2 = args[1]; + String fout = args[2]; + + // open the input files + HipoReader r1 = new HipoReader(); + HipoReader r2 = new HipoReader(); + r1.open(f1); + r2.open(f2); + + // grab schemas (dictionary) from run1 + SchemaFactory dict = r1.getSchemaFactory(); + + // prepare output writer with same dictionary + HipoWriter w = new HipoWriter(); + w.getSchemaFactory().copy(dict); + w.open(fout); + + Event e1 = new Event(); + Event e2 = new Event(); + Event eOut = new Event(); + + long n1 = r1.getEventCount(); + long n2 = r2.getEventCount(); + long nEvents = Math.min(n1, n2); + + // loop over events + for (int ievt = 0; ievt < nEvents; ievt++) { + + r1.getEvent(e1, ievt); + r2.getEvent(e2, ievt); + + eOut.reset(); + + // loop over every schema (bank definition) in file1 + for (Schema schema : dict.getSchemaList()) { + + // only diff this bank if BOTH events actually have it + if (!e1.hasBank(schema) || !e2.hasBank(schema)) { + continue; + } + + // build banks for this schema + Bank b1 = new Bank(schema); + Bank b2 = new Bank(schema); + + // read bank content from each event + e1.read(b1); + e2.read(b2); + + int rows = Math.min(b1.getRows(), b2.getRows()); + + // create output (diff) bank with same schema + Bank bDiff = new Bank(schema); + bDiff.setRows(rows); // coatjava 13.0.2 Bank has setRows() + + // loop rows/cols and subtract numeric columns + int ncols = b1.getSchema().getElements(); + for (int row = 0; row < rows; row++) { + + for (int col = 0; col < ncols; col++) { + int ftype = b1.getSchema().getType(col); + String fname = b1.getSchema().getElementName(col); + + switch (ftype) { + // NOTE: mapping here matches coatjava's internal codes + // 1 = byte (int8) + // 2 = short (int16) + // 3 = int (int32) + // 4 = float (float32) + // 5 = double(float64) + // 6 = long (int64) + case 1: // byte + bDiff.putByte(fname, row, + (byte)(b2.getByte(fname,row) - b1.getByte(fname,row))); + break; + case 2: // short + bDiff.putShort(fname, row, + (short)(b2.getShort(fname,row) - b1.getShort(fname,row))); + break; + case 3: // int + bDiff.putInt(fname, row, + b2.getInt(fname,row) - b1.getInt(fname,row)); + break; + case 4: // float + bDiff.putFloat(fname, row, + b2.getFloat(fname,row) - b1.getFloat(fname,row)); + break; + case 5: // double + bDiff.putDouble(fname, row, + b2.getDouble(fname,row) - b1.getDouble(fname,row)); + break; + case 6: // long + bDiff.putLong(fname, row, + b2.getLong(fname,row) - b1.getLong(fname,row)); + break; + default: + // unhandled type (arrays, etc.) -> just copy run2's value + copyFieldNoSub(ftype, fname, row, b2, bDiff); + break; + } + } // end col loop + } // end row loop + + // attach this diff bank into the output event + eOut.write(bDiff); + } // end schema loop + + // write the diff event + w.addEvent(eOut); + } // end event loop + + w.close(); + r1.close(); + r2.close(); + } + + // fallback copy logic for non-subtracted types + private static void copyFieldNoSub(int ftype, String fname, int row, + Bank src, Bank dst) { + + switch (ftype) { + case 1: // byte + dst.putByte(fname, row, src.getByte(fname,row)); + break; + case 2: // short + dst.putShort(fname, row, src.getShort(fname,row)); + break; + case 3: // int + dst.putInt(fname, row, src.getInt(fname,row)); + break; + case 4: // float + dst.putFloat(fname, row, src.getFloat(fname,row)); + break; + case 5: // double + dst.putDouble(fname, row, src.getDouble(fname,row)); + break; + case 6: // long + dst.putLong(fname, row, src.getLong(fname,row)); + break; + default: + // arrays / strings / etc. + // We don't currently have a generic copier for those in this script. + // It's ok to skip: they just won't appear in the diff bank. + break; + } + } +} + +