Skip to content

Commit 6bf93ea

Browse files
rvibracRomain Vibrac
rvibrac
and
Romain Vibrac
authored
Fix bug on join with hidden column (#15)
* Fix bug on join with hidden column * Fix tests Co-authored-by: Romain Vibrac <romain.vibrac@datasweet.fr>
1 parent b823c9e commit 6bf93ea

6 files changed

Lines changed: 213 additions & 40 deletions

File tree

‎export.go‎

Lines changed: 33 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,44 @@
11
package datatable
22

3+
// ExportOptions to add options for exporting (like showing hidden columns)
4+
type ExportOptions struct {
5+
WithHiddenCols bool
6+
}
7+
8+
type ExportOption func(*ExportOptions)
9+
10+
// ExportHidden to show a column when exporting (default false)
11+
func ExportHidden(v bool) ExportOption {
12+
return func(opts *ExportOptions) {
13+
opts.WithHiddenCols = v
14+
}
15+
}
16+
17+
// newExportOptions to build the ExportOptions in order to acces the parameters
18+
func newExportOptions(opt ...ExportOption) ExportOptions {
19+
var opts ExportOptions
20+
for _, o := range opt {
21+
o(&opts)
22+
}
23+
return opts
24+
25+
}
26+
327
// ToMap to export the datatable to a json-like struct
4-
func (t *DataTable) ToMap() []map[string]interface{} {
28+
func (t *DataTable) ToMap(opt ...ExportOption) []map[string]interface{} {
529
if t == nil {
630
return nil
731
}
832

33+
opts := newExportOptions(opt...)
934
if err := t.evaluateExpressions(); err != nil {
1035
panic(err)
1136
}
1237

1338
// visible columns
1439
cols := make(map[string]int)
1540
for i, col := range t.cols {
16-
if col.IsVisible() {
41+
if opts.WithHiddenCols || col.IsVisible() {
1742
cols[col.Name()] = i
1843
}
1944
}
@@ -30,11 +55,12 @@ func (t *DataTable) ToMap() []map[string]interface{} {
3055
}
3156

3257
// ToTable to export the datatable to a csv-like struct
33-
func (t *DataTable) ToTable() [][]interface{} {
58+
func (t *DataTable) ToTable(opt ...ExportOption) [][]interface{} {
3459
if t == nil {
3560
return nil
3661
}
3762

63+
opts := newExportOptions(opt...)
3864
if err := t.evaluateExpressions(); err != nil {
3965
panic(err)
4066
}
@@ -45,14 +71,13 @@ func (t *DataTable) ToTable() [][]interface{} {
4571
var headers []interface{}
4672
var cols []int
4773
for i, col := range t.cols {
48-
if col.IsVisible() {
74+
if opts.WithHiddenCols || col.IsVisible() {
4975
cols = append(cols, i)
5076
headers = append(headers, col.Name())
5177
}
5278
}
5379

5480
rows = append(rows, headers)
55-
5681
for i := 0; i < t.nrows; i++ {
5782
r := make([]interface{}, 0, len(cols))
5883
for _, pos := range cols {
@@ -76,11 +101,12 @@ type SchemaColumn struct {
76101
}
77102

78103
// ToSchema to export the datatable to a schema struct
79-
func (t *DataTable) ToSchema() *Schema {
104+
func (t *DataTable) ToSchema(opt ...ExportOption) *Schema {
80105
if t == nil {
81106
return nil
82107
}
83108

109+
opts := newExportOptions(opt...)
84110
if err := t.evaluateExpressions(); err != nil {
85111
panic(err)
86112
}
@@ -93,7 +119,7 @@ func (t *DataTable) ToSchema() *Schema {
93119
// visible columns
94120
var cols []int
95121
for i, col := range t.cols {
96-
if col.IsVisible() {
122+
if opts.WithHiddenCols || col.IsVisible() {
97123
cols = append(cols, i)
98124
schema.Columns = append(schema.Columns, SchemaColumn{Type: col.UnderlyingType().Name(), Name: col.Name()})
99125
}

‎export_test.go‎

Lines changed: 51 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -59,16 +59,30 @@ func TestToTable(t *testing.T) {
5959
assert.NotNil(t, out)
6060

6161
expected := `[
62-
["Client ID", "Nom", "email", "ville"],
63-
[1, "Aimée MARECHAL", "aime.marechal@example.com", "Paris"],
64-
[2, "Esmée LEFORT", "esmee.lefort@example.com", "Lyon"],
65-
[3, "Marine PREVOST", "m.prevost@example.com", "Lille"],
66-
[4, "Luc ROLLAND", "lucrolland@example.com", "Marseille"]
67-
]`
62+
["Client ID", "Nom", "email", "ville"],
63+
[1, "Aimée MARECHAL", "aime.marechal@example.com", "Paris"],
64+
[2, "Esmée LEFORT", "esmee.lefort@example.com", "Lyon"],
65+
[3, "Marine PREVOST", "m.prevost@example.com", "Lille"],
66+
[4, "Luc ROLLAND", "lucrolland@example.com", "Marseille"]
67+
]`
6868

6969
bytes, err := json.Marshal(out)
7070
assert.NoError(t, err)
7171
assert.JSONEq(t, expected, string(bytes))
72+
73+
out2 := dt.ToTable(datatable.ExportHidden(true))
74+
assert.NotNil(t, out2)
75+
76+
expected2 := `[
77+
["Client ID", "prenom", "nom", "Nom", "email", "ville"],
78+
[1, "Aimée", "Marechal", "Aimée MARECHAL", "aime.marechal@example.com", "Paris"],
79+
[2, "Esmée", "Lefort", "Esmée LEFORT", "esmee.lefort@example.com", "Lyon"],
80+
[3, "Marine", "Prevost", "Marine PREVOST", "m.prevost@example.com", "Lille"],
81+
[4, "Luc", "Rolland", "Luc ROLLAND", "lucrolland@example.com", "Marseille"]
82+
]`
83+
bytes, err = json.Marshal(out2)
84+
assert.NoError(t, err)
85+
assert.JSONEq(t, expected2, string(bytes))
7286
}
7387

7488
func TestToMap(t *testing.T) {
@@ -86,6 +100,20 @@ func TestToMap(t *testing.T) {
86100
bytes, err := json.Marshal(out)
87101
assert.NoError(t, err)
88102
assert.JSONEq(t, expected, string(bytes))
103+
104+
out2 := dt.ToMap(datatable.ExportHidden(true))
105+
assert.NotNil(t, out2)
106+
107+
expected2 := `[
108+
{ "Client ID":1, "prenom": "Aimée", "nom": "Marechal", "Nom":"Aimée MARECHAL", "email":"aime.marechal@example.com", "ville":"Paris" },
109+
{ "Client ID":2, "prenom": "Esmée", "nom": "Lefort", "Nom":"Esmée LEFORT", "email":"esmee.lefort@example.com", "ville":"Lyon" },
110+
{ "Client ID":3, "prenom": "Marine", "nom": "Prevost", "Nom":"Marine PREVOST", "email":"m.prevost@example.com", "ville":"Lille" },
111+
{ "Client ID":4, "prenom": "Luc", "nom": "Rolland", "Nom":"Luc ROLLAND", "email":"lucrolland@example.com", "ville":"Marseille" }
112+
]`
113+
114+
bytes, err = json.Marshal(out2)
115+
assert.NoError(t, err)
116+
assert.JSONEq(t, expected2, string(bytes))
89117
}
90118

91119
func TestToSchema(t *testing.T) {
@@ -104,4 +132,21 @@ func TestToSchema(t *testing.T) {
104132
assert.Equal(t, []interface{}{2, "Esmée LEFORT", "esmee.lefort@example.com", "Lyon"}, schema.Rows[1])
105133
assert.Equal(t, []interface{}{3, "Marine PREVOST", "m.prevost@example.com", "Lille"}, schema.Rows[2])
106134
assert.Equal(t, []interface{}{4, "Luc ROLLAND", "lucrolland@example.com", "Marseille"}, schema.Rows[3])
135+
136+
schema2 := dt.ToSchema(datatable.ExportHidden(true))
137+
assert.NotNil(t, schema2)
138+
assert.Equal(t, "Customers", schema2.Name)
139+
assert.Equal(t, []datatable.SchemaColumn{
140+
datatable.SchemaColumn{"Client ID", "NullInt"},
141+
datatable.SchemaColumn{"prenom", "NullString"},
142+
datatable.SchemaColumn{"nom", "NullString"},
143+
datatable.SchemaColumn{"Nom", "NullString"},
144+
datatable.SchemaColumn{"email", "NullString"},
145+
datatable.SchemaColumn{"ville", "NullString"},
146+
}, schema2.Columns)
147+
assert.Len(t, schema2.Rows, 4)
148+
assert.Equal(t, []interface{}{1, "Aimée", "Marechal", "Aimée MARECHAL", "aime.marechal@example.com", "Paris"}, schema2.Rows[0])
149+
assert.Equal(t, []interface{}{2, "Esmée", "Lefort", "Esmée LEFORT", "esmee.lefort@example.com", "Lyon"}, schema2.Rows[1])
150+
assert.Equal(t, []interface{}{3, "Marine", "Prevost", "Marine PREVOST", "m.prevost@example.com", "Lille"}, schema2.Rows[2])
151+
assert.Equal(t, []interface{}{4, "Luc", "Rolland", "Luc ROLLAND", "lucrolland@example.com", "Marseille"}, schema2.Rows[3])
107152
}

‎join.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -236,8 +236,8 @@ func (j *joinImpl) checkInput() error {
236236
func (j *joinImpl) initColMapper() {
237237
mcols := make(map[string][]string)
238238
for _, t := range j.tables {
239-
for _, name := range t.Columns() {
240-
mcols[name] = append(mcols[name], t.Name())
239+
for _, name := range t.cols {
240+
mcols[name.name] = append(mcols[name.name], t.Name())
241241
}
242242
}
243243
j.mcols = mcols
@@ -306,14 +306,14 @@ func (j *joinImpl) join(left, right *DataTable) (*DataTable, error) {
306306
join.initHashTable()
307307

308308
// Copy rows
309-
for _, refrow := range ref.table.Rows() {
309+
for _, refrow := range ref.table.Rows(ExportHidden(true)) {
310310
// Create hash
311311
hash := hasher.Row(refrow, ref.on)
312312

313313
// Have we same hash in jointable ?
314314
if indexes, ok := join.hashtable[hash]; ok {
315315
for _, idx := range indexes {
316-
joinrow := join.table.Row(idx)
316+
joinrow := join.table.Row(idx, ExportHidden(true))
317317
row := out.NewRow()
318318
for _, cm := range ref.cmapper {
319319
row[cm[1]] = refrow.Get(cm[0])
@@ -332,6 +332,7 @@ func (j *joinImpl) join(left, right *DataTable) (*DataTable, error) {
332332
out.Append(row)
333333
}
334334
}
335+
// out.Print(os.Stdout, PrintColumnType(false))
335336

336337
// Outer: we must copy rows not consummed in right (join) table
337338
if j.mode == outerJoin {

‎join_test.go‎

Lines changed: 55 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ func sampleForJoin() (*datatable.DataTable, *datatable.DataTable) {
1717
customers.AddColumn("nom", datatable.String)
1818
customers.AddColumn("email", datatable.String)
1919
customers.AddColumn("ville", datatable.String)
20+
// customers.AddColumn("concat", datatable.String, datatable.Expr("CONCAT(`prenom`,`nom`)"))
2021
customers.AppendRow(1, "Aimée", "Marechal", "aime.marechal@example.com", "Paris")
2122
customers.AppendRow(2, "Esmée", "Lefort", "esmee.lefort@example.com", "Lyon")
2223
customers.AppendRow(3, "Marine", "Prevost", "m.prevost@example.com", "Lille")
@@ -59,17 +60,16 @@ func TestJoinOn(t *testing.T) {
5960

6061
func TestInnerJoin(t *testing.T) {
6162
customers, orders := sampleForJoin()
62-
63+
customers.AddColumn("concat", datatable.String, datatable.Expr("concat(`prenom`, `nom`)"))
6364
dt, err := customers.InnerJoin(orders, datatable.On("[Customers].[id]", "[Orders].[user_id]"))
6465
assert.NoError(t, err)
6566
assert.NotNil(t, dt)
66-
6767
checkTable(t, dt,
68-
"id", "prenom", "nom", "email", "ville", "date_achat", "num_facture", "prix_total",
69-
1, "Aimée", "Marechal", "aime.marechal@example.com", "Paris", time.Date(2013, time.January, 23, 0, 0, 0, 0, time.UTC), "A00103", 203.14,
70-
1, "Aimée", "Marechal", "aime.marechal@example.com", "Paris", time.Date(2013, time.February, 14, 0, 0, 0, 0, time.UTC), "A00104", 124.00,
71-
2, "Esmée", "Lefort", "esmee.lefort@example.com", "Lyon", time.Date(2013, time.February, 17, 0, 0, 0, 0, time.UTC), "A00105", 149.45,
72-
3, "Marine", "Prevost", "m.prevost@example.com", "Lille", time.Date(2013, time.February, 21, 0, 0, 0, 0, time.UTC), "A00106", 235.35,
68+
"id", "prenom", "nom", "email", "ville", "concat", "date_achat", "num_facture", "prix_total",
69+
1, "Aimée", "Marechal", "aime.marechal@example.com", "Paris", "AiméeMarechal", time.Date(2013, time.January, 23, 0, 0, 0, 0, time.UTC), "A00103", 203.14,
70+
1, "Aimée", "Marechal", "aime.marechal@example.com", "Paris", "AiméeMarechal", time.Date(2013, time.February, 14, 0, 0, 0, 0, time.UTC), "A00104", 124.00,
71+
2, "Esmée", "Lefort", "esmee.lefort@example.com", "Lyon", "EsméeLefort", time.Date(2013, time.February, 17, 0, 0, 0, 0, time.UTC), "A00105", 149.45,
72+
3, "Marine", "Prevost", "m.prevost@example.com", "Lille", "MarinePrevost", time.Date(2013, time.February, 21, 0, 0, 0, 0, time.UTC), "A00106", 235.35,
7373
)
7474
}
7575

@@ -125,27 +125,63 @@ func TestOuterJoin(t *testing.T) {
125125
)
126126
}
127127

128-
func TestJoinWithExpr(t *testing.T) {
128+
func TestInnerJoinWithExprOnHidden(t *testing.T) {
129129
customers, orders := sampleForJoin()
130-
customers.AddColumn("upper_ville", datatable.String, datatable.Expr("UPPER(ville)"))
130+
customers.AddColumn("id2", datatable.Int, datatable.Expr("`id`+100"))
131+
orders.AddColumn("user_id2", datatable.Int, datatable.Expr("`user_id`+100"))
132+
customers.HideColumn("id")
133+
dt, err := customers.InnerJoin(orders, datatable.On("[Customers].[id2]", "[Orders].[user_id2]"))
134+
assert.NoError(t, err)
135+
assert.NotNil(t, dt)
131136

132-
dt, err := customers.InnerJoin(orders, datatable.On("[Customers].[id]", "[Orders].[user_id]"))
137+
fmt.Println(dt)
138+
139+
checkTable(t, dt,
140+
"prenom", "nom", "email", "ville", "id2", "user_id", "date_achat", "num_facture", "prix_total",
141+
"Aimée", "Marechal", "aime.marechal@example.com", "Paris", 101, 1, time.Date(2013, time.January, 23, 0, 0, 0, 0, time.UTC), "A00103", 203.14,
142+
"Aimée", "Marechal", "aime.marechal@example.com", "Paris", 101, 1, time.Date(2013, time.February, 14, 0, 0, 0, 0, time.UTC), "A00104", 124.00,
143+
"Esmée", "Lefort", "esmee.lefort@example.com", "Lyon", 102, 2, time.Date(2013, time.February, 17, 0, 0, 0, 0, time.UTC), "A00105", 149.45,
144+
"Marine", "Prevost", "m.prevost@example.com", "Lille", 103, 3, time.Date(2013, time.February, 21, 0, 0, 0, 0, time.UTC), "A00106", 235.35,
145+
)
146+
}
147+
148+
func TestLeftJoinWithExpr(t *testing.T) {
149+
customers, orders := sampleForJoin()
150+
customers.AddColumn("id2", datatable.Int, datatable.Expr("`id`+100"))
151+
orders.AddColumn("user_id2", datatable.Int, datatable.Expr("`user_id`+100"))
152+
dt, err := customers.LeftJoin(orders, datatable.On("[Customers].[id2]", "[Orders].[user_id2]"))
133153
assert.NoError(t, err)
134154
assert.NotNil(t, dt)
135155

136-
col := dt.Column("upper_ville")
137-
assert.Equal(t, datatable.String, col.Type())
138-
assert.Equal(t, "NullString", col.UnderlyingType().Name())
139-
assert.True(t, col.IsComputed())
156+
fmt.Println(dt)
157+
158+
checkTable(t, dt,
159+
"id", "prenom", "nom", "email", "ville", "id2", "user_id", "date_achat", "num_facture", "prix_total",
160+
1, "Aimée", "Marechal", "aime.marechal@example.com", "Paris", 101, 1, time.Date(2013, time.January, 23, 0, 0, 0, 0, time.UTC), "A00103", 203.14,
161+
1, "Aimée", "Marechal", "aime.marechal@example.com", "Paris", 101, 1, time.Date(2013, time.February, 14, 0, 0, 0, 0, time.UTC), "A00104", 124.00,
162+
2, "Esmée", "Lefort", "esmee.lefort@example.com", "Lyon", 102, 2, time.Date(2013, time.February, 17, 0, 0, 0, 0, time.UTC), "A00105", 149.45,
163+
3, "Marine", "Prevost", "m.prevost@example.com", "Lille", 103, 3, time.Date(2013, time.February, 21, 0, 0, 0, 0, time.UTC), "A00106", 235.35,
164+
4, "Luc", "Rolland", "lucrolland@example.com", "Marseille", 104, nil, nil, nil, nil,
165+
)
166+
}
167+
168+
func TestRightJoinWithExpr(t *testing.T) {
169+
customers, orders := sampleForJoin()
170+
customers.AddColumn("id2", datatable.Int, datatable.Expr("`id`+100"))
171+
orders.AddColumn("user_id2", datatable.Int, datatable.Expr("`user_id`+100"))
172+
dt, err := customers.RightJoin(orders, datatable.On("[Customers].[id2]", "[Orders].[user_id2]"))
173+
assert.NoError(t, err)
174+
assert.NotNil(t, dt)
140175

141176
fmt.Println(dt)
142177

143178
checkTable(t, dt,
144-
"id", "prenom", "nom", "email", "ville", "upper_ville", "date_achat", "num_facture", "prix_total",
145-
1, "Aimée", "Marechal", "aime.marechal@example.com", "Paris", "PARIS", time.Date(2013, time.January, 23, 0, 0, 0, 0, time.UTC), "A00103", 203.14,
146-
1, "Aimée", "Marechal", "aime.marechal@example.com", "Paris", "PARIS", time.Date(2013, time.February, 14, 0, 0, 0, 0, time.UTC), "A00104", 124.00,
147-
2, "Esmée", "Lefort", "esmee.lefort@example.com", "Lyon", "LYON", time.Date(2013, time.February, 17, 0, 0, 0, 0, time.UTC), "A00105", 149.45,
148-
3, "Marine", "Prevost", "m.prevost@example.com", "Lille", "LILLE", time.Date(2013, time.February, 21, 0, 0, 0, 0, time.UTC), "A00106", 235.35,
179+
"id", "prenom", "nom", "email", "ville", "id2", "user_id", "date_achat", "num_facture", "prix_total",
180+
1, "Aimée", "Marechal", "aime.marechal@example.com", "Paris", 101, 1, time.Date(2013, time.January, 23, 0, 0, 0, 0, time.UTC), "A00103", 203.14,
181+
1, "Aimée", "Marechal", "aime.marechal@example.com", "Paris", 101, 1, time.Date(2013, time.February, 14, 0, 0, 0, 0, time.UTC), "A00104", 124.00,
182+
2, "Esmée", "Lefort", "esmee.lefort@example.com", "Lyon", 102, 2, time.Date(2013, time.February, 17, 0, 0, 0, 0, time.UTC), "A00105", 149.45,
183+
3, "Marine", "Prevost", "m.prevost@example.com", "Lille", 103, 3, time.Date(2013, time.February, 21, 0, 0, 0, 0, time.UTC), "A00106", 235.35,
184+
nil, nil, nil, nil, nil, nil, 5, time.Date(2013, time.March, 2, 0, 0, 0, 0, time.UTC), "A00107", 47.58,
149185
)
150186
}
151187

‎table.go‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -115,19 +115,20 @@ func (t *DataTable) Records() [][]string {
115115

116116
// Rows returns the rows in datatable
117117
// Computes all expressions.
118-
func (t *DataTable) Rows() []Row {
118+
func (t *DataTable) Rows(opt ...ExportOption) []Row {
119119
if t == nil {
120120
return nil
121121
}
122122

123+
opts := newExportOptions(opt...)
123124
if err := t.evaluateExpressions(); err != nil {
124125
panic(err)
125126
}
126127

127128
// visible columns
128129
cols := make(map[string]int)
129130
for i, col := range t.cols {
130-
if col.IsVisible() {
131+
if opts.WithHiddenCols || col.IsVisible() {
131132
cols[col.Name()] = i
132133
}
133134
}
@@ -150,11 +151,14 @@ func (t *DataTable) String() string {
150151
}
151152

152153
// Row gets the row at index
153-
func (t *DataTable) Row(at int) Row {
154+
func (t *DataTable) Row(at int, opt ...ExportOption) Row {
155+
opts := newExportOptions(opt...)
154156
t.evaluateExpressions()
155157
r := make(Row, len(t.cols))
156158
for _, col := range t.cols {
157-
r[col.name] = col.serie.Get(at)
159+
if opts.WithHiddenCols || col.IsVisible() {
160+
r[col.name] = col.serie.Get(at)
161+
}
158162
}
159163
return r
160164
}

0 commit comments

Comments
 (0)