From bf96a614ba305025f6c447e5898efddb92cca49a Mon Sep 17 00:00:00 2001 From: Dmitry Kusakin <66311786+Dmitry-Ku@users.noreply.github.com> Date: Tue, 23 Nov 2021 16:04:03 +0300 Subject: [PATCH 1/5] gpbackup should be faster without compression https://arenadata.atlassian.net/browse/PFP-217 gpbackup should be faster in 1.5 times without compression --- backup/data.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/backup/data.go b/backup/data.go index 7f92cdcde..5b56b91e2 100644 --- a/backup/data.go +++ b/backup/data.go @@ -72,7 +72,11 @@ func CopyTableOut(connectionPool *dbconn.DBConn, table Table, destinationToWrite sendToDestinationCommand = fmt.Sprintf("| %s backup_data %s", pluginConfig.ExecutablePath, pluginConfig.ConfigPath) } - copyCommand := fmt.Sprintf("PROGRAM '%s%s %s %s'", checkPipeExistsCommand, customPipeThroughCommand, sendToDestinationCommand, destinationToWrite) + if customPipeThroughCommand == "cat -" { + copyCommand := fmt.Sprintf("'%s'", destinationToWrite) + } else { + copyCommand := fmt.Sprintf("PROGRAM '%s%s %s %s'", checkPipeExistsCommand, customPipeThroughCommand, sendToDestinationCommand, destinationToWrite) + } query := fmt.Sprintf("COPY %s TO %s WITH CSV DELIMITER '%s' ON SEGMENT IGNORE EXTERNAL PARTITIONS;", table.FQN(), copyCommand, tableDelim) gplog.Verbose("Worker %d: %s", connNum, query) From 0bc81e88aa7dbca90b8d794d69ce834307337864 Mon Sep 17 00:00:00 2001 From: Dmitry Kusakin <66311786+Dmitry-Ku@users.noreply.github.com> Date: Tue, 23 Nov 2021 17:37:26 +0300 Subject: [PATCH 2/5] Variable has been declared in function scope visibility https://arenadata.atlassian.net/browse/PFP-217 gpbackup should be faster in 1.5 times without compression --- backup/data.go | 1 + 1 file changed, 1 insertion(+) diff --git a/backup/data.go b/backup/data.go index 5b56b91e2..7329586d7 100644 --- a/backup/data.go +++ b/backup/data.go @@ -59,6 +59,7 @@ func CopyTableOut(connectionPool *dbconn.DBConn, table Table, destinationToWrite checkPipeExistsCommand := "" customPipeThroughCommand := utils.GetPipeThroughProgram().OutputCommand sendToDestinationCommand := ">" + copyCommand := "" if MustGetFlagBool(options.SINGLE_DATA_FILE) { /* * The segment TOC files are always written to the segment data directory for From c0eed92f30525a071776f799a8f581664a0a846e Mon Sep 17 00:00:00 2001 From: Dmitry Kusakin <66311786+Dmitry-Ku@users.noreply.github.com> Date: Tue, 23 Nov 2021 17:56:13 +0300 Subject: [PATCH 3/5] Fixed assignment error https://arenadata.atlassian.net/browse/PFP-217 gpbackup should be faster in 1.5 times without compression --- backup/data.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backup/data.go b/backup/data.go index 7329586d7..186983868 100644 --- a/backup/data.go +++ b/backup/data.go @@ -74,9 +74,9 @@ func CopyTableOut(connectionPool *dbconn.DBConn, table Table, destinationToWrite } if customPipeThroughCommand == "cat -" { - copyCommand := fmt.Sprintf("'%s'", destinationToWrite) + copyCommand = fmt.Sprintf("'%s'", destinationToWrite) } else { - copyCommand := fmt.Sprintf("PROGRAM '%s%s %s %s'", checkPipeExistsCommand, customPipeThroughCommand, sendToDestinationCommand, destinationToWrite) + copyCommand = fmt.Sprintf("PROGRAM '%s%s %s %s'", checkPipeExistsCommand, customPipeThroughCommand, sendToDestinationCommand, destinationToWrite) } query := fmt.Sprintf("COPY %s TO %s WITH CSV DELIMITER '%s' ON SEGMENT IGNORE EXTERNAL PARTITIONS;", table.FQN(), copyCommand, tableDelim) From 42d8c1c0c925d4a1e09607a9cd9965ba2abbab3d Mon Sep 17 00:00:00 2001 From: Dmitry Kusakin <66311786+Dmitry-Ku@users.noreply.github.com> Date: Wed, 24 Nov 2021 12:27:15 +0300 Subject: [PATCH 4/5] 1 test has been updated Test is "will back up a table to its own file without compression". --- backup/data_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backup/data_test.go b/backup/data_test.go index ffb0b31fa..736362201 100644 --- a/backup/data_test.go +++ b/backup/data_test.go @@ -121,7 +121,7 @@ var _ = Describe("backup/data tests", func() { }) It("will back up a table to its own file without compression", func() { utils.SetPipeThroughProgram(utils.PipeThroughProgram{Name: "cat", OutputCommand: "cat -", InputCommand: "cat -", Extension: ""}) - execStr := regexp.QuoteMeta("COPY public.foo TO PROGRAM 'cat - > /backups/20170101/20170101010101/gpbackup__20170101010101_3456' WITH CSV DELIMITER ',' ON SEGMENT IGNORE EXTERNAL PARTITIONS;") + execStr := regexp.QuoteMeta("COPY public.foo TO '/backups/20170101/20170101010101/gpbackup__20170101010101_3456' WITH CSV DELIMITER ',' ON SEGMENT IGNORE EXTERNAL PARTITIONS;") mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 0)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456" From d90755146d4c93c6ff6286bd7d07bccd16e095ad Mon Sep 17 00:00:00 2001 From: Dmitry Kusakin <66311786+Dmitry-Ku@users.noreply.github.com> Date: Wed, 24 Nov 2021 12:38:45 +0300 Subject: [PATCH 5/5] Logic creating of copy command changed Single file option and plugin using are bypassing --- backup/data.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backup/data.go b/backup/data.go index 186983868..dd7ca97ea 100644 --- a/backup/data.go +++ b/backup/data.go @@ -71,11 +71,11 @@ func CopyTableOut(connectionPool *dbconn.DBConn, table Table, destinationToWrite customPipeThroughCommand = "cat -" } else if MustGetFlagString(options.PLUGIN_CONFIG) != "" { sendToDestinationCommand = fmt.Sprintf("| %s backup_data %s", pluginConfig.ExecutablePath, pluginConfig.ConfigPath) + } else if customPipeThroughCommand == "cat -" { + copyCommand = fmt.Sprintf("'%s'", destinationToWrite) } - if customPipeThroughCommand == "cat -" { - copyCommand = fmt.Sprintf("'%s'", destinationToWrite) - } else { + if copyCommand == "" { copyCommand = fmt.Sprintf("PROGRAM '%s%s %s %s'", checkPipeExistsCommand, customPipeThroughCommand, sendToDestinationCommand, destinationToWrite) }