BCP Out performance

include/freetds/checks.h

@@ -57,8 +57,8 @@ void tds_check_freeze_extra(const TDSFREEZE * freeze);
 #  define TDS_MARK_UNDEFINED(ptr, len) do {} while(0)
 #endif
 
+void tds_extra_assert_check(const char* fn, int line, int cond, const char* cond_str);
 #if ENABLE_EXTRA_CHECKS
-void tds_extra_assert_check(const char *fn, int line, int cond, const char *cond_str);
 #  define tds_extra_assert(cond) \
 	tds_extra_assert_check(__FILE__, __LINE__, cond, #cond)
 #else

include/freetds/convert.h

@@ -91,6 +91,14 @@ TDS_INT tds_convert(const TDSCONTEXT *context, int srctype, const void *src, TDS
 
 size_t tds_strftime(char *buf, size_t maxsize, const char *format, const TDSDATEREC * timeptr, int prec);
 
+/* Fast int to string (massively outperforms sprintf in hot loop).
+ * No null termination; returns number of characters read.
+ */
+size_t tds_u32toafast_right(char out[10], uint32_t v);
+size_t tds_u32toafast(char out[10], uint32_t v);
+size_t tds_i32toafast(char out[11],  int32_t v);
+void   tds_02dfast(char out[2], int v);	/* v must be 0-99 (non negative) */
+
 #ifdef __cplusplus
 #if 0
 {

include/freetds/stream.h

@@ -24,6 +24,8 @@
 #error Include tds.h first
 #endif
 
+#include <stdio.h>		/* FILE * */
+#include <freetds/iconv.h>	/* TDS_ICONV_DIRECTION */
 #include <freetds/pushvis.h>
 
 /** define a stream of data used for input */
@@ -51,6 +53,23 @@ typedef struct tds_output_stream {
 	size_t buf_len;
 } TDSOUTSTREAM;
 
+/** Output stream that writes to file.
+ * Does not open/close fp; and fp can be NULL.
+ */
+typedef struct tds_fileout_stream {
+	TDSOUTSTREAM stream;
+	FILE* fp;
+	char block[4096];
+	/** How to buffer the data -- _IONBF, _IOLBF or _IOFBF (default). 
+	 * In _IOLBF line will still be flushed on reaching block size. */
+	int buff_type;
+} TDSFILEOUTSTREAM;
+
+void tds_fileout_stream_init(TDSFILEOUTSTREAM* s, FILE* fp, int buff_type);
+
+TDSRET tds_fileout_stream_flush(TDSFILEOUTSTREAM* s);
+TDSRET tds_fileout_stream_put(TDSFILEOUTSTREAM* s, void const *src, size_t n);
+
 /** Convert a stream from istream to ostream using a specific conversion */
 TDSRET tds_convert_stream(TDSSOCKET * tds, TDSICONV * char_conv, TDS_ICONV_DIRECTION direction,
 	TDSINSTREAM * istream, TDSOUTSTREAM *ostream);

include/freetds/tds.h

@@ -693,7 +693,7 @@ struct tds_column
 					 */
 	TDS_TINYINT column_varint_size;	/**< size of length when reading from wire (0, 1, 2 or 4) */
 
-	TDS_TINYINT column_prec;	/**< precision for decimal/numeric */
+	TDS_TINYINT column_prec;	/**< precision for decimal/numeric and some datetime types */
 	TDS_TINYINT column_scale;	/**< scale for decimal/numeric */
 
 	struct

src/dblib/bcp.c

@@ -41,9 +41,11 @@
 #endif
 
 #include <freetds/tds.h>
+#include <freetds/checks.h>
 #include <freetds/iconv.h>
 #include <freetds/convert.h>
 #include <freetds/bytes.h>
+#include <freetds/stream.h>
 #include <freetds/utils/string.h>
 #include <freetds/encodings.h>
 #include <freetds/replacements.h>
@@ -52,6 +54,8 @@
 #include <syberror.h>
 #include <dblib.h>
 
+#define BCP_DEFAULT_DATEFMT "%Y-%m-%d %H:%M:%S.%z"
+
 #define HOST_COL_CONV_ERROR 1
 #define HOST_COL_NULL_ERROR 2
 
@@ -85,6 +89,7 @@ static int rtrim_u16(uint16_t *str, int len, uint16_t space);
 static STATUS _bcp_read_hostfile(DBPROCESS * dbproc, FILE * hostfile, bool *row_error, bool skip);
 static int _bcp_readfmt_colinfo(DBPROCESS * dbproc, char *buf, BCP_HOSTCOLINFO * ci);
 static int _bcp_get_term_var(const BYTE * pdata, const BYTE * term, int term_len);
+static int _bcp_strftime(TDS_UCHAR* buf, size_t maxsize, const char* format, const TDSDATEREC* dr, int prec);
 
 /*
  * "If a host file is being used ... the default data formats are as follows:
@@ -765,9 +770,17 @@ _bcp_convert_out(DBPROCESS * dbproc, TDSCOLUMN *curcol, BCP_HOSTCOLINFO *hostcol
 	if (is_datetime_type(srctype) && is_ascii_type(hostcol->datatype)) {
 		TDSDATEREC when;
 
+		/* Some date/time types have variable precision (e.g. MS datetime2),
+		 * and set curcol->column_prec with that precision.
+		 * DATETIME is printed with precision 3 by MS BCP & ASE BCP however
+		 * curcol->column_prec is not set for that type.
+		 * (It might be better to set column_prec at load time for all
+		 * of the datetime types...)
+		 */
+		int prec = curcol->column_prec ? curcol->column_prec : 3;
+
 		tds_datecrack(srctype, src, &when);
-		buflen = (int)tds_strftime((TDS_CHAR *)(*p_data), 256,
-					 bcpdatefmt, &when, 3);
+		buflen = _bcp_strftime(*p_data, 256, bcpdatefmt, &when, prec);
 	} else if (srclen == 0 && is_variable_type(curcol->column_type)
 		   && is_ascii_type(hostcol->datatype)) {
 		/*
@@ -847,7 +860,7 @@ bcp_cache_prefix_len(BCP_HOSTCOLINFO *hostcol, const TDSCOLUMN *curcol)
 }
 
 static RETCODE
-bcp_write_prefix(FILE *hostfile, BCP_HOSTCOLINFO *hostcol, TDSCOLUMN *curcol, int buflen)
+bcp_write_prefix(TDSFILEOUTSTREAM *hoststream, BCP_HOSTCOLINFO *hostcol, TDSCOLUMN *curcol, int buflen)
 {
 	union {
 		TDS_TINYINT ti;
@@ -874,10 +887,7 @@ bcp_write_prefix(FILE *hostfile, BCP_HOSTCOLINFO *hostcol, TDSCOLUMN *curcol, in
 		u.li = buflen;
 		break;
 	}
-	if (fwrite(&u, plen, 1, hostfile) == 1)
-		return SUCCEED;
-
-	return FAIL;
+	return TDS_SUCCEED(tds_fileout_stream_put(hoststream, &u, plen)) ? SUCCEED : FAIL;
 }
 
 /**
@@ -894,7 +904,8 @@ bcp_write_prefix(FILE *hostfile, BCP_HOSTCOLINFO *hostcol, TDSCOLUMN *curcol, in
 static RETCODE
 _bcp_exec_out(DBPROCESS * dbproc, DBINT * rows_copied)
 {
-	FILE *hostfile = NULL;
+	TDSFILEOUTSTREAM hoststream;
+	FILE* hostfp = NULL;
 	TDS_UCHAR *data = NULL;
 	int i;
 
@@ -918,9 +929,13 @@ _bcp_exec_out(DBPROCESS * dbproc, DBINT * rows_copied)
 	tds = dbproc->tds_socket;
 	assert(tds);
 
+	/* See if they want a BCP date format different from the default.
+	 * The ASE bcp client does use the locale for bcp out; but all bcp clients
+	 * appear to successfully read this format for input, regardless of locale.
+	 */
 	bcpdatefmt = getenv("FREEBCP_DATEFMT");
-	if (!bcpdatefmt)
-		bcpdatefmt = "%Y-%m-%d %H:%M:%S.%z";
+	if (bcpdatefmt && !strcmp(bcpdatefmt, BCP_DEFAULT_DATEFMT))
+		bcpdatefmt = NULL;
 
 	if (dbproc->bcpinfo->direction == DB_QUERYOUT ) {
 		if (TDS_FAILED(tds_submit_query(tds, tds_dstr_cstr(&dbproc->bcpinfo->tablename))))
@@ -957,11 +972,11 @@ _bcp_exec_out(DBPROCESS * dbproc, DBINT * rows_copied)
 	 * TODO above we allocate many buffer just to convert and store
 	 * to file.. avoid all that passages...
 	 */
-
-	if (!(hostfile = fopen(dbproc->hostfileinfo->hostfile, "w"))) {
+	if (!(hostfp = fopen(dbproc->hostfileinfo->hostfile, "w"))) {
 		dbperror(dbproc, SYBEBCUO, errno);
 		goto Cleanup;
 	}
+	tds_fileout_stream_init(&hoststream, hostfp, _IOFBF);
 
 	/* fetch a row of data from the server */
 
@@ -997,7 +1012,7 @@ _bcp_exec_out(DBPROCESS * dbproc, DBINT * rows_copied)
 			}
 
 			/* The prefix */
-			if (bcp_write_prefix(hostfile, hostcol, curcol, buflen) != SUCCEED)
+			if (bcp_write_prefix(&hoststream, hostcol, curcol, buflen) != SUCCEED)
 				goto write_error;
 
 			/* The data */
@@ -1006,23 +1021,25 @@ _bcp_exec_out(DBPROCESS * dbproc, DBINT * rows_copied)
 			}
 
 			if (buflen > 0) {
-				if (fwrite(data, buflen, 1, hostfile) != 1)
+				if ( !TDS_SUCCEED(tds_fileout_stream_put(&hoststream, data, buflen)) )
 					goto write_error;
 			}
 
 			/* The terminator */
 			if (hostcol->terminator && hostcol->term_len > 0) {
-				if (fwrite(hostcol->terminator, hostcol->term_len, 1, hostfile) != 1)
+				if ( !TDS_SUCCEED(tds_fileout_stream_put(&hoststream, hostcol->terminator, hostcol->term_len)) )
 					goto write_error;
 			}
 		}
 		rows_written++;
 	}
-	if (fclose(hostfile) != 0) {
+
+	tds_fileout_stream_flush(&hoststream);
+	if (hostfp && fclose(hostfp) != 0) {
 		dbperror(dbproc, SYBEBCUC, errno);
 		goto Cleanup;
 	}
-	hostfile = NULL;
+	hostfp = NULL;
 
 	if (row_of_query + 1 < dbproc->hostfileinfo->firstrow) {
 		/*
@@ -1043,8 +1060,8 @@ _bcp_exec_out(DBPROCESS * dbproc, DBINT * rows_copied)
 	dbperror(dbproc, SYBEBCWE, errno);
 
 Cleanup:
-	if (hostfile)
-		fclose(hostfile);
+	if (hostfp)
+		fclose(hostfp);
 	free(data);
 	return FAIL;
 }
@@ -2421,3 +2438,59 @@ _bcp_free_storage(DBPROCESS * dbproc)
 	dbproc->bcpinfo = NULL;
 }
 
+/** Fast strftime for default bcp format */
+static int _bcp_strftime(TDS_UCHAR* buf, size_t maxsize, const char* format, const TDSDATEREC* dr, int prec)
+{
+	char* cbuf = (char*)buf;
+
+	if (format)
+		return (int)tds_strftime(cbuf, maxsize, format, dr, prec);
+
+	cbuf += tds_u32toafast(cbuf, dr->year);
+
+	*cbuf++ = '-';
+	tds_02dfast(cbuf, dr->month + 1);
+	cbuf += 2;
+
+	*cbuf++ = '-';
+	tds_02dfast(cbuf, dr->day);
+	cbuf += 2;
+
+	*cbuf++ = ' ';
+	tds_02dfast(cbuf, dr->hour);
+	cbuf += 2;
+
+	*cbuf++ = ':';
+	tds_02dfast(cbuf, dr->minute);
+	cbuf += 2;
+
+	*cbuf++ = ':';
+	tds_02dfast(cbuf, dr->second);
+	cbuf += 2;
+
+	if (prec > 0)
+	{
+		char ibuf[10];
+		*cbuf++ = '.';
+
+		/* Mask off a negative sign (input should not contain this anyway) */
+		memset(ibuf, '0', tds_u32toafast_right(ibuf, dr->decimicrosecond & 0x7FFFFFFF));
+		/* Skip first 3 digits here, as decimicrosecond should not exceed 9,999,999 */
+		memcpy(cbuf, ibuf + 3, prec);
+		cbuf += prec;
+	}
+	*cbuf = '\0';
+
+#if ENABLE_EXTRA_CHECKS
+	{
+		char tbuf[256];
+		tds_strftime(tbuf, sizeof tbuf, BCP_DEFAULT_DATEFMT, dr, prec);
+		if (strcmp(tbuf, (char*)buf))
+		{
+			fprintf(stderr, "_bcp_strftime(%s) does not match tds_strftime(%s)\n", tbuf, (char *)buf);
+			tds_extra_assert(!"_bcp_strftime mismatch");
+		}
+	}
+#endif
+	return (int)(cbuf - (char*)buf);
+}