ExportAllData ignores groups/tuples for grouped data

[Matthias1373]

Currently, when exporting all data, wpd ignores the groups and tuples column and omits it everywhere.

May I suggest to rewrite the corresponding generateCSV function?

function generateCSV() {
    wpd.popup.close('export-all-data-popup');
    // generate file and trigger download

    // loop over all datasets
    let plotData = wpd.appData.getPlotData();
    let dsColl = plotData.getDatasets();

    if (dsColl == null || dsColl.length === 0) {
        // axes is not aligned, show an error message?
        wpd.messagePopup.show(wpd.gettext('no-datasets-to-export-error'),
            wpd.gettext('no-datasets-to-export'));
        return;
    }

    let maxDatapts = 0;
    let header = [];
    let varheader = [];
    let valData = [];
    let numCols = 0;

    for (let i = 0; i < dsColl.length; i++) {
        let axes = plotData.getAxesForDataset(dsColl[i]);
        if (axes == null)
            continue;
        let axLab = axes.getAxesLabels();
        let axdims = axLab.length;
        numCols += axdims;
        let pts = dsColl[i].getCount();
        if (pts > maxDatapts) {
            maxDatapts = pts;
        }
        header.push(dsColl[i].name);
+      Pseudocode:
+     if ( dsColl[i] has pointGroups):
+            colNb = axdims + 2
-        for (let j = 0; j < axdims; j++) {
+       for (let j = 0; j < colNb; j++) {     
            if (j !== 0) {
                header.push('');
            }
+         if j<axdims:
              varheader.push(axLab[j]);
+         else if j==axdims:
+              varheader.push("Tuple");
+         else if j==axdims+1:
+             varheader.push("Group");
        }



    for (let i = 0; i < maxDatapts; i++) {
        var valRow = [];
        for (let j = 0; j < numCols; j++) {
            valRow.push('');
        }
        valData.push(valRow);
    }

    let colIdx = 0;
    for (let i = 0; i < dsColl.length; i++) {
        let axes = plotData.getAxesForDataset(dsColl[i]);
        if (axes == null)
            continue;
        let axLab = axes.getAxesLabels();
        let axdims = axLab.length;
        let pts = dsColl[i].getCount();
        for (let j = 0; j < pts; j++) {
            let px = dsColl[i].getPixel(j);
            let val = getValueAtPixel(j, axes, px);
            for (let di = 0; di < axdims; di++) {
                valData[j][colIdx + di] = val[di];
            }
+      if dsColl[i] has pointGroups:
+          let tuple = dsColl[i].getTuple();
+          let group = dsColl[i].getGroup();
+          valData[j][colIdx + axdims] = tuple;
+          valData[j][colIdx + axdims + 1] = group;
        }
        colIdx += axdims;
    }

    let csvText = header.join(',') + '\n' + varheader.join(',') + '\n';
    for (let i = 0; i < maxDatapts; i++) {
        csvText += valData[i].join(',') + '\n';
    }

    // download
    wpd.download.csv(csvText, "wpd_datasets.csv");
}

While I'm at it, I'd like to thank everybody involved in development very much for this tool, I've been using it for ages now :D

[Matthias1373]

I was blocked and needed a quick fix, so overriding the javascript definition of generateCSV locally with the following suggestion fixed my problem. This should be easily translatable to the codebase definition. If I get some time on my hands, I'll turn it into a pull request.

generateCSV: function() {
        wpd.popup.close("export-all-data-popup");
        var s = wpd.appData.getPlotData()
          , o = s.getDatasets();
        if (null == o || 0 === o.length)
            wpd.messagePopup.show(wpd.gettext("no-datasets-to-export-error"), wpd.gettext("no-datasets-to-export"));
        else {
            let t = 0;
            var r = []
              , l = []
              , d = [];
            let a = 0;
            for (let e = 0; e < o.length; e++) {
                var p = s.getAxesForDataset(o[e]);
                if (null != p) {
                    var g = p.getAxesLabels()
                      , c = g.length
                      , z = o[e].getPointGroups()
                      , p = (a += c,
                    o[e].getCount());
                    p > t && (t = p),
                    r.push(o[e].name);
                    if (z.length>0) {
                        c += 2;
                    }
                    for (let e = 0; e < c; e++) {
                        if (e !== 0) {
                            r.push("");
                        }
                        if (e <= 1) {
                            l.push(g[e]);
                        } else if (e == 2) {
                            l.push("Tuple");
                        } else if (e == 3) {
                            l.push("Group");
                        }
                    }
                }
            }
            for (let e = 0; e < t; e++) {
                var u = [];
                for (let e = 0; e < a; e++)
                    u.push("");
                d.push(u)
            }
            let i = 0;
            for (let e = 0; e < o.length; e++) {
                var h = s.getAxesForDataset(o[e]);
                if (null != h) {
                    var m = h.getAxesLabels().length
                      , w = o[e].getCount()
                      , z = o[e].getPointGroups() ;
                    for (let t = 0; t < w; t++) {
                        var x = o[e].getPixel(t)
                          , f = function(e, t, a) {
                            var i = t.pixelToData(a.x, a.y);
                            if (t instanceof wpd.XYAxes)
                                for (var n, s = 0; s <= 1; s++)
                                    t.isDate(s) && (n = t.getInitialDateFormat(s),
                                    i[s] = wpd.dateConverter.formatDateNumber(i[s], n));
                            else
                                t instanceof wpd.CircularChartRecorderAxes ? i[0] = wpd.dateConverter.formatDateNumber(i[0], t.getTimeFormat()) : t instanceof wpd.BarAxes && (i = ["", i[0]],
                                null == a.metadata ? i[0] = "Bar" + e : i[0] = a.metadata[0]);
                            return i
                        }(t, h, x);
                        for (let e = 0; e < m; e++)
                            d[t][i + e] = f[e]
                    }
                    i += m; 
                    if (z.length>0){
                        console.log("New Dataset");
                        for (let j = 0; j < z.length; j++) {
                            var x = o[e].getPixelIndexesInGroup(j);
                            console.log(x);
                            
                            for (let k = 0; k < x.length; k++) {
                                console.log(x[k],i+m,j);
                                console.log(x[k],i+m+1,z[j]);
                                d[x[k]][i+m-2] = k;
                                d[x[k]][i+m+1-2] = z[j];
                            };
                        };
                        i += 2;
                    }
                }
            }
            let n = r.join(",") + "\n" + l.join(",") + "\n";
            for (let e = 0; e < t; e++)
                n += d[e].join(",") + "\n";
            wpd.download.csv(n, "wpd_datasets.csv")
        }

[ankitrohatgi]

Hi, thanks for pointing this out. I think not many users use the point groups feature so it was missed. I can incorporate in the next update in a couple of days.

[Matthias1373]

If you are at it, could you also support Overrides for the final export? :)
I have a very rudimentary solution in js that only includes the YOverride column as Overrides, but having both in the final output would be great! :)

I'm sure this can be solved more elegantly, but maybe it gives some inspiration:

generateCSV: function() {
        wpd.popup.close("export-all-data-popup");
        var s = wpd.appData.getPlotData()
          , o = s.getDatasets();
        if (null == o || 0 === o.length)
            wpd.messagePopup.show(wpd.gettext("no-datasets-to-export-error"), wpd.gettext("no-datasets-to-export"));
        else {
            let t = 0;
            var r = []
              , l = []
              , d = []
              , markerOverrides = 0;
            let a = 0;
            for (let e = 0; e < o.length; e++) {
                var p = s.getAxesForDataset(o[e]);
                if (null != p) {
                    var g = p.getAxesLabels()
                      , c = g.length
                      , z = o[e].getPointGroups()
                      , y = o[e].getMetadataKeys()
                      , p = (a += c,
                    o[e].getCount());
                    p > t && (t = p),
                    r.push(o[e].name);
                    if(y.length>0){
                        c += 1; 
                        markerOverrides = 1;
                    }
                    if (z.length>0) {
                        c += 2;
                    }
                    for (let e = 0; e < c; e++) {
                        if (e !== 0) {
                            r.push("");
                        }
                        if (e <= 1) {
                            l.push(g[e]);
                        } else if (e == 2 && markerOverrides == 0) {
                            l.push("Tuple");
                        } else if (e == 3 && markerOverrides == 0) {
                            l.push("Group");
                        } else if (e == 2 && markerOverrides == 1) {
                            l.push("Override");
                        } else if (e == 3 && markerOverrides == 1) {
                            l.push("Tuple");
                        } else if (e == 4 && markerOverrides == 1) {
                            l.push("Group");
                        }
                    }
                    markerOverrides = 0;
                }
            }
            for (let e = 0; e < t; e++) {
                var u = [];
                for (let e = 0; e < a; e++)
                    u.push("");
                d.push(u)
            }
            let i = 0;
            for (let e = 0; e < o.length; e++) {
                var h = s.getAxesForDataset(o[e]);
                if (null != h) {
                    var m = h.getAxesLabels().length
                      , w = o[e].getCount()
                      , z = o[e].getPointGroups() ;
                    for (let t = 0; t < w; t++) {
                        var x = o[e].getPixel(t)
                          , f = function(e, t, a) {
                            var i = t.pixelToData(a.x, a.y);
                            if (t instanceof wpd.XYAxes)
                                for (var n, s = 0; s <= 1; s++)
                                    t.isDate(s) && (n = t.getInitialDateFormat(s),
                                    i[s] = wpd.dateConverter.formatDateNumber(i[s], n));
                            else
                                t instanceof wpd.CircularChartRecorderAxes ? i[0] = wpd.dateConverter.formatDateNumber(i[0], t.getTimeFormat()) : t instanceof wpd.BarAxes && (i = ["", i[0]],
                                null == a.metadata ? i[0] = "Bar" + e : i[0] = a.metadata[0]);
                            return i
                        }(t, h, x);
                        for (let e = 0; e < m; e++)
                            d[t][i + e] = f[e]
                        if (null != x.metadata && null != x.metadata.overrides && null != x.metadata.overrides.y){
                            yoverride = x.metadata.overrides.y;
                            d[t][i+m] = yoverride;
                        }
                    }
                    if(o[e].getMetadataKeys().length>0){
                        m+=1;
                    }
                    i += m; 
                    if (z.length>0){
                        for (let j = 0; j < z.length; j++) {
                            var x = o[e].getPixelIndexesInGroup(j)
                                , shift = 2;
                            if(o[e].getMetadataKeys().length>0){
                                shift += 1;
                            }
                            for (let k = 0; k < x.length; k++) {
                                d[x[k]][i+m-shift] = k;
                                d[x[k]][i+m+1-shift] = z[j];
                            };
                        };
                        i += 2;
                    }
                }
            }
            let n = r.join(",") + "\n" + l.join(",") + "\n";
            for (let e = 0; e < t; e++)
                n += d[e].join(",") + "\n";
            wpd.download.csv(n, "wpd_datasets.csv")
        }
    },

[clairecouty]

Hi everyone here, I would like to upvote this issue since I have been facing the exact same problem.
Thanks a lot for what you are doing here.