From dcb8cafac9c5e632d57f0198f53b8a72c4b523fb Mon Sep 17 00:00:00 2001 From: sbhawani Date: Wed, 28 Jan 2026 12:44:03 -0500 Subject: [PATCH 1/9] [Draft] Adds: hipo diff writer from banks in 2 hipo files --- bin/hipo-diffBanks | 10 ++ .../java/org/jlab/utils/HipoDiffBanks.java | 167 ++++++++++++++++++ 2 files changed, 177 insertions(+) create mode 100755 bin/hipo-diffBanks create mode 100644 common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiffBanks.java diff --git a/bin/hipo-diffBanks b/bin/hipo-diffBanks new file mode 100755 index 0000000000..af938f6d13 --- /dev/null +++ b/bin/hipo-diffBanks @@ -0,0 +1,10 @@ +#!/bin/bash + +. `dirname $0`/../libexec/env.sh + +export MALLOC_ARENA_MAX=1 + +java ${JAVA_OPTS-} -Xmx1536m -Xms1024m -XX:+UseSerialGC \ + -cp ${COATJAVA_CLASSPATH:-''} \ + org.jlab.utils.HipoDiffBanks \ + $* diff --git a/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiffBanks.java b/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiffBanks.java new file mode 100644 index 0000000000..2e24eb63b7 --- /dev/null +++ b/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiffBanks.java @@ -0,0 +1,167 @@ +package org.jlab.utils; + +import org.jlab.jnp.hipo4.io.HipoReader; +import org.jlab.jnp.hipo4.io.HipoWriter; +import org.jlab.jnp.hipo4.data.Event; +import org.jlab.jnp.hipo4.data.Bank; +import org.jlab.jnp.hipo4.data.Schema; +import org.jlab.jnp.hipo4.data.SchemaFactory; + +public class HipoDiffBanks { + + public static void main(String[] args) throws Exception { + if (args.length < 3) { + System.err.println("usage: HipoDiffBanks run1.hipo run2.hipo out.hipo"); + System.exit(1); + } + + String f1 = args[0]; + String f2 = args[1]; + String fout = args[2]; + + // open the input files + HipoReader r1 = new HipoReader(); + HipoReader r2 = new HipoReader(); + r1.open(f1); + r2.open(f2); + + // grab schemas (dictionary) from run1 + SchemaFactory dict = r1.getSchemaFactory(); + + // prepare output writer with same dictionary + HipoWriter w = new HipoWriter(); + w.getSchemaFactory().copy(dict); + w.open(fout); + + Event e1 = new Event(); + Event e2 = new Event(); + Event eOut = new Event(); + + long n1 = r1.getEventCount(); + long n2 = r2.getEventCount(); + long nEvents = Math.min(n1, n2); + + // loop over events + for (int ievt = 0; ievt < nEvents; ievt++) { + + r1.getEvent(e1, ievt); + r2.getEvent(e2, ievt); + + eOut.reset(); + + // loop over every schema (bank definition) in file1 + for (Schema schema : dict.getSchemaList()) { + + // only diff this bank if BOTH events actually have it + if (!e1.hasBank(schema) || !e2.hasBank(schema)) { + continue; + } + + // build banks for this schema + Bank b1 = new Bank(schema); + Bank b2 = new Bank(schema); + + // read bank content from each event + e1.read(b1); + e2.read(b2); + + int rows = Math.min(b1.getRows(), b2.getRows()); + + // create output (diff) bank with same schema + Bank bDiff = new Bank(schema); + bDiff.setRows(rows); // coatjava 13.0.2 Bank has setRows() + + // loop rows/cols and subtract numeric columns + int ncols = b1.getSchema().getElements(); + for (int row = 0; row < rows; row++) { + + for (int col = 0; col < ncols; col++) { + int ftype = b1.getSchema().getType(col); + String fname = b1.getSchema().getElementName(col); + + switch (ftype) { + // NOTE: mapping here matches coatjava's internal codes + // 1 = byte (int8) + // 2 = short (int16) + // 3 = int (int32) + // 4 = float (float32) + // 5 = double(float64) + // 6 = long (int64) + case 1: // byte + bDiff.putByte(fname, row, + (byte)(b2.getByte(fname,row) - b1.getByte(fname,row))); + break; + case 2: // short + bDiff.putShort(fname, row, + (short)(b2.getShort(fname,row) - b1.getShort(fname,row))); + break; + case 3: // int + bDiff.putInt(fname, row, + b2.getInt(fname,row) - b1.getInt(fname,row)); + break; + case 4: // float + bDiff.putFloat(fname, row, + b2.getFloat(fname,row) - b1.getFloat(fname,row)); + break; + case 5: // double + bDiff.putDouble(fname, row, + b2.getDouble(fname,row) - b1.getDouble(fname,row)); + break; + case 6: // long + bDiff.putLong(fname, row, + b2.getLong(fname,row) - b1.getLong(fname,row)); + break; + default: + // unhandled type (arrays, etc.) -> just copy run2's value + copyFieldNoSub(ftype, fname, row, b2, bDiff); + break; + } + } // end col loop + } // end row loop + + // attach this diff bank into the output event + eOut.write(bDiff); + } // end schema loop + + // write the diff event + w.addEvent(eOut); + } // end event loop + + w.close(); + r1.close(); + r2.close(); + } + + // fallback copy logic for non-subtracted types + private static void copyFieldNoSub(int ftype, String fname, int row, + Bank src, Bank dst) { + + switch (ftype) { + case 1: // byte + dst.putByte(fname, row, src.getByte(fname,row)); + break; + case 2: // short + dst.putShort(fname, row, src.getShort(fname,row)); + break; + case 3: // int + dst.putInt(fname, row, src.getInt(fname,row)); + break; + case 4: // float + dst.putFloat(fname, row, src.getFloat(fname,row)); + break; + case 5: // double + dst.putDouble(fname, row, src.getDouble(fname,row)); + break; + case 6: // long + dst.putLong(fname, row, src.getLong(fname,row)); + break; + default: + // arrays / strings / etc. + // We don't currently have a generic copier for those in this script. + // It's ok to skip: they just won't appear in the diff bank. + break; + } + } +} + + From 9bbc38aad99d1b048f06f2302d806544ff898e38 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Thu, 29 Jan 2026 19:34:41 -0500 Subject: [PATCH 2/9] switch to return values --- .../main/java/org/jlab/utils/HipoDiff.java | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java b/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java index 52f82c4e2e..5c6f43bc4b 100644 --- a/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java +++ b/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java @@ -107,7 +107,8 @@ public static void main(String args[]) { compare(readerA, readerB); } - public static void compare(HipoReader a, HipoReader b) { + public static int compare(HipoReader a, HipoReader b) { + int ret=0; Event eventA = new Event(); Event eventB = new Event(); while (a.hasNext() && b.hasNext() && (nmax < 1 || nevent < nmax)) { @@ -115,7 +116,7 @@ public static void compare(HipoReader a, HipoReader b) { a.nextEvent(eventA); b.nextEvent(eventB); eventA.read(runConfigBank); - compare(eventA, eventB); + ret += compare(eventA, eventB); } System.out.println("\n Analyzed " + nevent + " with " + nbadevent + " bad banks"); System.out.println(nbadrow + "/" + nrow + " mismatched rows"); @@ -124,17 +125,21 @@ public static void compare(HipoReader a, HipoReader b) { System.out.println(name + " " + badEntries.get(name)); } System.exit(nbadevent + nbadrow + nbadentry); + return ret; } - public static void compare(Event a, Event b) { + public static int compare(Event a, Event b) { + int ret = 0; for (int i = 0; i < banksA.size(); i++) { a.read(banksA.get(i)); b.read(banksB.get(i)); - compare(banksA.get(i), banksB.get(i)); + ret += compare(banksA.get(i), banksB.get(i)); } + return ret; } - public static void compare(SortedBank a, SortedBank b) { + public static int compare(SortedBank a, SortedBank b) { + int ret=0; if (a.getRows() != b.getRows()) { System.out.println("========================= Different number of rows:"); runConfigBank.show(); @@ -142,7 +147,7 @@ public static void compare(SortedBank a, SortedBank b) { b.show(); nbadevent++; System.out.println("========================="); - return; + return ++ret; } int[] rowsA = sortIndex == null ? null : a.getSorted(sortIndex); int[] rowsB = sortIndex == null ? null : b.getSorted(sortIndex); @@ -207,11 +212,13 @@ public static void compare(SortedBank a, SortedBank b) { m.put(elementName, 0); } m.put(elementName, m.get(elementName) + 1); + ret++; } } if (mismatch) { nbadrow++; } } + return 0; } } From a26ffe67ef53ddda231f3d27c5a99244a805f309 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Thu, 29 Jan 2026 19:37:14 -0500 Subject: [PATCH 3/9] add writer --- .../src/main/java/org/jlab/utils/HipoDiff.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java b/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java index 5c6f43bc4b..c5ee5402bd 100644 --- a/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java +++ b/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java @@ -7,6 +7,7 @@ import org.jlab.jnp.hipo4.data.Event; import org.jlab.jnp.hipo4.data.Schema; import org.jlab.jnp.hipo4.data.SchemaFactory; +import org.jlab.jnp.hipo4.io.HipoWriter; import org.jlab.utils.options.OptionParser; public class HipoDiff { @@ -57,6 +58,7 @@ int[] getSorted(int... index) { static boolean verboseMode; static boolean quietMode; static Bank runConfigBank; + static HipoWriter writer; static ArrayList banksA = new ArrayList<>(); static ArrayList banksB = new ArrayList<>(); @@ -70,6 +72,7 @@ public static void main(String args[]) { op.addOption("-Q", null, "verbose mode"); op.addOption("-b", null, "name of bank to diff"); op.addOption("-s", null, "sort on column index"); + op.addOption("-o", null, "output HIPO diff file"); op.setRequiresInputList(true); op.parse(args); if (op.getInputList().size() != 2) { @@ -104,6 +107,12 @@ public static void main(String args[]) { banksB.add(new SortedBank(sf.getSchema(op.getOption("-b").stringValue()))); } + if (op.getOption("-o").stringValue() != null) { + writer = new HipoWriter(); + writer.getSchemaFactory().copy(readerA.getSchemaFactory()); + writer.open(op.getOption("-o").stringValue()); + } + compare(readerA, readerB); } @@ -133,7 +142,12 @@ public static int compare(Event a, Event b) { for (int i = 0; i < banksA.size(); i++) { a.read(banksA.get(i)); b.read(banksB.get(i)); - ret += compare(banksA.get(i), banksB.get(i)); + int x = compare(banksA.get(i), banksB.get(i)); + if (x != 0 && writer != null) { + // FIXME: need the new diff event here + writer.addEvent(a); + } + ret += x; } return ret; } From 518da791485c94a8be733b474f51b353bf4b26e6 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Thu, 29 Jan 2026 20:28:52 -0500 Subject: [PATCH 4/9] extract and use a diff bank method --- .../main/java/org/jlab/utils/HipoDiff.java | 44 +++++++++++++++++-- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java b/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java index c5ee5402bd..f8ce70b176 100644 --- a/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java +++ b/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java @@ -43,6 +43,41 @@ int[] getSorted(int... index) { } return rows; } + public Bank getDiff(Bank b) { + Bank diff = new Bank(getSchema()); + int rows = Math.min(getRows(), b.getRows()); + int ncols = getSchema().getElements(); + for (int row = 0; row < rows; row++) { + for (int col = 0; col < ncols; col++) { + int ftype = getSchema().getType(col); + String fname = getSchema().getElementName(col); + switch (ftype) { + case 1: // byte + diff.putByte(fname, row, (byte)(b.getByte(fname,row) - getByte(fname,row))); + break; + case 2: // short + diff.putShort(fname, row, (short)(b.getShort(fname,row) - getShort(fname,row))); + break; + case 3: // int + diff.putInt(fname, row, b.getInt(fname,row) - getInt(fname,row)); + break; + case 4: // float + diff.putFloat(fname, row, b.getFloat(fname,row) - getFloat(fname,row)); + break; + case 5: // double + diff.putDouble(fname, row, b.getDouble(fname,row) - getDouble(fname,row)); + break; + case 6: // long + diff.putLong(fname, row, b.getLong(fname,row) - getLong(fname,row)); + break; + default: + // unhandled type (arrays, etc.) + break; + } + } + } + return diff; + } } static int nrow = 0; @@ -59,6 +94,7 @@ int[] getSorted(int... index) { static boolean quietMode; static Bank runConfigBank; static HipoWriter writer; + static Event event; static ArrayList banksA = new ArrayList<>(); static ArrayList banksB = new ArrayList<>(); @@ -142,12 +178,12 @@ public static int compare(Event a, Event b) { for (int i = 0; i < banksA.size(); i++) { a.read(banksA.get(i)); b.read(banksB.get(i)); - int x = compare(banksA.get(i), banksB.get(i)); - if (x != 0 && writer != null) { - // FIXME: need the new diff event here + if (writer != null) { + event.reset(); + event.write(banksA.get(i).getDiff(banksB.get(i))); writer.addEvent(a); } - ret += x; + ret += compare(banksA.get(i), banksB.get(i)); } return ret; } From d8b310209a2df500f1e8bee0a12b6f9eaae1c646 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Thu, 29 Jan 2026 20:47:38 -0500 Subject: [PATCH 5/9] rename SortedBank to DiffBank --- .../main/java/org/jlab/utils/HipoDiff.java | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java b/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java index f8ce70b176..78a13b0edd 100644 --- a/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java +++ b/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java @@ -15,8 +15,8 @@ public class HipoDiff { /** * Bank sortable by any integer columns. */ - static class SortedBank extends Bank { - SortedBank(Schema s) { super(s); } + static class DiffBank extends Bank { + DiffBank(Schema s) { super(s); } /** * @param index the bank column indices to sort on * @return the sorted row indices @@ -43,6 +43,10 @@ int[] getSorted(int... index) { } return rows; } + /** + * @param b the bank to compare with + * @return the resulting diff bank + */ public Bank getDiff(Bank b) { Bank diff = new Bank(getSchema()); int rows = Math.min(getRows(), b.getRows()); @@ -96,8 +100,8 @@ public Bank getDiff(Bank b) { static HipoWriter writer; static Event event; - static ArrayList banksA = new ArrayList<>(); - static ArrayList banksB = new ArrayList<>(); + static ArrayList banksA = new ArrayList<>(); + static ArrayList banksB = new ArrayList<>(); static HashMap> badEntries = new HashMap<>(); public static void main(String args[]) { @@ -135,12 +139,12 @@ public static void main(String args[]) { if (op.getOption("-b").stringValue() == null) { for (Schema s : sf.getSchemaList()) { - banksA.add(new SortedBank(s)); - banksB.add(new SortedBank(s)); + banksA.add(new DiffBank(s)); + banksB.add(new DiffBank(s)); } } else { - banksA.add(new SortedBank(sf.getSchema(op.getOption("-b").stringValue()))); - banksB.add(new SortedBank(sf.getSchema(op.getOption("-b").stringValue()))); + banksA.add(new DiffBank(sf.getSchema(op.getOption("-b").stringValue()))); + banksB.add(new DiffBank(sf.getSchema(op.getOption("-b").stringValue()))); } if (op.getOption("-o").stringValue() != null) { @@ -188,7 +192,7 @@ public static int compare(Event a, Event b) { return ret; } - public static int compare(SortedBank a, SortedBank b) { + public static int compare(DiffBank a, DiffBank b) { int ret=0; if (a.getRows() != b.getRows()) { System.out.println("========================= Different number of rows:"); From 171377914a9a4ddf04ec8ff1b6f0456fa776b483 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Thu, 29 Jan 2026 20:48:40 -0500 Subject: [PATCH 6/9] remove string-based bank access --- .../src/main/java/org/jlab/utils/HipoDiff.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java b/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java index 78a13b0edd..8a942381c8 100644 --- a/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java +++ b/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java @@ -54,25 +54,24 @@ public Bank getDiff(Bank b) { for (int row = 0; row < rows; row++) { for (int col = 0; col < ncols; col++) { int ftype = getSchema().getType(col); - String fname = getSchema().getElementName(col); switch (ftype) { case 1: // byte - diff.putByte(fname, row, (byte)(b.getByte(fname,row) - getByte(fname,row))); + diff.putByte(col, row, (byte)(b.getByte(col,row) - getByte(col,row))); break; case 2: // short - diff.putShort(fname, row, (short)(b.getShort(fname,row) - getShort(fname,row))); + diff.putShort(col, row, (short)(b.getShort(col,row) - getShort(col,row))); break; case 3: // int - diff.putInt(fname, row, b.getInt(fname,row) - getInt(fname,row)); + diff.putInt(col, row, b.getInt(col,row) - getInt(col,row)); break; case 4: // float - diff.putFloat(fname, row, b.getFloat(fname,row) - getFloat(fname,row)); + diff.putFloat(col, row, b.getFloat(col,row) - getFloat(col,row)); break; case 5: // double - diff.putDouble(fname, row, b.getDouble(fname,row) - getDouble(fname,row)); + diff.putDouble(col, row, b.getDouble(col,row) - getDouble(col,row)); break; case 6: // long - diff.putLong(fname, row, b.getLong(fname,row) - getLong(fname,row)); + diff.putLong(col, row, b.getLong(col,row) - getLong(col,row)); break; default: // unhandled type (arrays, etc.) From 684daf4bacac3ad00387eb6263fc2d49a7800eef Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Thu, 29 Jan 2026 20:49:21 -0500 Subject: [PATCH 7/9] cleanup --- .../clas-io/src/main/java/org/jlab/utils/HipoDiff.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java b/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java index 8a942381c8..2903b7500f 100644 --- a/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java +++ b/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java @@ -53,8 +53,7 @@ public Bank getDiff(Bank b) { int ncols = getSchema().getElements(); for (int row = 0; row < rows; row++) { for (int col = 0; col < ncols; col++) { - int ftype = getSchema().getType(col); - switch (ftype) { + switch (getSchema().getType(col)) { case 1: // byte diff.putByte(col, row, (byte)(b.getByte(col,row) - getByte(col,row))); break; From aa6a53cc46bdf3c5f73fa0818c2761c904ff59ab Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Thu, 29 Jan 2026 20:54:09 -0500 Subject: [PATCH 8/9] close the writer --- common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java b/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java index 2903b7500f..9fccb87e7a 100644 --- a/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java +++ b/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java @@ -152,6 +152,8 @@ public static void main(String args[]) { } compare(readerA, readerB); + + if (writer != null) writer.close(); } public static int compare(HipoReader a, HipoReader b) { From 1ea04b4c40d9f1e480468d13b0d53ec3ed66bdf6 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Thu, 29 Jan 2026 20:55:01 -0500 Subject: [PATCH 9/9] actually write the diff event --- common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java b/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java index 9fccb87e7a..8a06d076bd 100644 --- a/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java +++ b/common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java @@ -185,7 +185,7 @@ public static int compare(Event a, Event b) { if (writer != null) { event.reset(); event.write(banksA.get(i).getDiff(banksB.get(i))); - writer.addEvent(a); + writer.addEvent(event); } ret += compare(banksA.get(i), banksB.get(i)); }