From 8b388ed4e67da64842b70066c81549cc6f286541 Mon Sep 17 00:00:00 2001 From: Benjamin Young Date: Sat, 12 Nov 2011 10:08:21 -0500 Subject: [PATCH 1/5] add utf8 encoding of string values --- sql2couchdb.php | 1 + 1 file changed, 1 insertion(+) diff --git a/sql2couchdb.php b/sql2couchdb.php index ba5df49..339be75 100755 --- a/sql2couchdb.php +++ b/sql2couchdb.php @@ -61,6 +61,7 @@ function castSQLToPHP($mysqlType, $value) case 'bool': $value = (bool) $value; break; case 'boolean': $value = (bool) $value; break; case 'tinyint': $value = (bool) $value; break; + default: $value = utf8_encode($value); break; } return $value; From 2f1f0bb65df291f8bb702b43dc7f209cdcec0bf8 Mon Sep 17 00:00:00 2001 From: Benjamin Young Date: Sat, 12 Nov 2011 10:08:36 -0500 Subject: [PATCH 2/5] fix missing $wasObject variable --- sql2couchdb.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sql2couchdb.php b/sql2couchdb.php index 339be75..33ba1af 100755 --- a/sql2couchdb.php +++ b/sql2couchdb.php @@ -69,6 +69,8 @@ function castSQLToPHP($mysqlType, $value) function map($row, $json, $mysqlQuery) { + $wasObject = false; + if($json) { //turn objects into associative arrays From 60dd436ecfe75a79cbdd85df4d85ab3ac2a0d5ca Mon Sep 17 00:00:00 2001 From: Benjamin Young Date: Sat, 12 Nov 2011 10:08:54 -0500 Subject: [PATCH 3/5] add batching of bulk writes 1000 at a time --- sql2couchdb.php | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/sql2couchdb.php b/sql2couchdb.php index 33ba1af..77f9ee6 100755 --- a/sql2couchdb.php +++ b/sql2couchdb.php @@ -209,10 +209,20 @@ function map($row, $json, $mysqlQuery) $docsToSend = array(); -while($row = mysql_fetch_array($mysqlQuery, MYSQL_ASSOC)) +$c = 0; +while($row = mysql_fetch_array($mysqlQuery, MYSQL_ASSOC)) { $docsToSend[] = map($row, clone $jsonFile->doc, $mysqlQuery); + $c++; + if ($c === 1001) { + $sag->bulk($docsToSend); + $docsToSend = array(); + $c = 0; + } +} -$sag->bulk($docsToSend); +if (count($docsToSend) > 0) { + $sag->bulk($docsToSend); +} mysql_free_result($mysqlQuery); mysql_close($mysqlCon); From e46de6ec0588ec6309f3123c8ba7c741c8a826da Mon Sep 17 00:00:00 2001 From: Benjamin Young Date: Mon, 14 Nov 2011 16:44:51 -0500 Subject: [PATCH 4/5] remove keys that have empty values returned from MySQL --- sql2couchdb.php | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/sql2couchdb.php b/sql2couchdb.php index 77f9ee6..a325e1b 100755 --- a/sql2couchdb.php +++ b/sql2couchdb.php @@ -84,7 +84,14 @@ function map($row, $json, $mysqlQuery) { foreach($json as $k => $v) { - $json[$k] = map($row, $v, $mysqlQuery); + $v = map($row, $v, $mysqlQuery); + if ($v === null) + { + unset($json[$k]); + continue; + } else { + $json[$k] = $v; + } if($k == '_id') $json[$k] = (string) $json[$k]; @@ -93,6 +100,8 @@ function map($row, $json, $mysqlQuery) elseif(is_string($json)) if($row[$json]) return castSQLToPHP(getColType($json, $mysqlQuery), $row[$json]); + else + return null; if($wasObject) $json = (object) $json; From 958bfc6492e1a8a1cdf4505a4593251f7eab88fb Mon Sep 17 00:00:00 2001 From: Benjamin Young Date: Mon, 5 Dec 2011 17:27:44 -0500 Subject: [PATCH 5/5] added --dump-here option to put $doc->_id.json files in a dir --- sql2couchdb.php | 64 +++++++++++++++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 23 deletions(-) diff --git a/sql2couchdb.php b/sql2couchdb.php index a325e1b..81fffd3 100755 --- a/sql2couchdb.php +++ b/sql2couchdb.php @@ -123,7 +123,8 @@ function map($row, $json, $mysqlQuery) "couchdb-pass:", "couchdb-db:", "couchdb-host:", - "couchdb-port:" + "couchdb-port:", + "dump-here:" ) ); @@ -137,6 +138,8 @@ function map($row, $json, $mysqlQuery) $couchdbConfig->host = '127.0.0.1'; $couchdbConfig->port = '5984'; +$dumpDir = false; + foreach($options as $opt => $value) { switch($opt) @@ -180,6 +183,10 @@ function map($row, $json, $mysqlQuery) case 'couchdb-port': $couchdbConfig->port = $value; break; + + case 'dump-here': + $dumpDir = $value; + break; } } @@ -196,16 +203,18 @@ function map($row, $json, $mysqlQuery) if(!$jsonFile->doc || !is_object($jsonFile->doc)) throw new Exception('Missing the doc.'); -// Set up Sag now so that we don't run the whole MySQL loop and then end up -// with an error. +if(!$dumpDir) { + // Set up Sag now so that we don't run the whole MySQL loop and then end up + // with an error. -$sag = new Sag($couchdbConfig->host, $couchdbConfig->port); -$sag->setDatabase($couchdbConfig->db); + $sag = new Sag($couchdbConfig->host, $couchdbConfig->port); + $sag->setDatabase($couchdbConfig->db); -if($couchdbConfig->user || $couchdbConfig->pass) - $sag->login($couchdbConfig->user, $couchdbConfig->pass); + if($couchdbConfig->user || $couchdbConfig->pass) + $sag->login($couchdbConfig->user, $couchdbConfig->pass); +} -// Get data from MySQL and send it to CouchDB. +// Get data from MySQL and send it to CouchDB or the $dumpDir if(!($mysqlCon = mysql_connect("{$mysqlConfig->host}:{$mysqlConfig->port}", $mysqlConfig->user, $mysqlConfig->pass))) throw new Exception('Unable to connect to MySQL: '.mysql_error()); @@ -216,21 +225,30 @@ function map($row, $json, $mysqlQuery) if(!($mysqlQuery = mysql_query($jsonFile->query, $mysqlCon))) throw new Exception('Invalid query: '.mysql_error()); -$docsToSend = array(); - -$c = 0; -while($row = mysql_fetch_array($mysqlQuery, MYSQL_ASSOC)) { - $docsToSend[] = map($row, clone $jsonFile->doc, $mysqlQuery); - $c++; - if ($c === 1001) { - $sag->bulk($docsToSend); - $docsToSend = array(); - $c = 0; - } -} - -if (count($docsToSend) > 0) { - $sag->bulk($docsToSend); +if(!$dumpDir) { + $docsToSend = array(); + + $c = 0; + while($row = mysql_fetch_array($mysqlQuery, MYSQL_ASSOC)) { + $docsToSend[] = map($row, clone $jsonFile->doc, $mysqlQuery); + $c++; + if ($c === 1001) { + $sag->bulk($docsToSend); + $docsToSend = array(); + $c = 0; + } + } + + if (count($docsToSend) > 0) { + $sag->bulk($docsToSend); + } +} else { + if(!file_exists($dumpDir)) + mkdir($dumpDir); + while($row = mysql_fetch_array($mysqlQuery, MYSQL_ASSOC)) { + $doc = map($row, clone $jsonFile->doc, $mysqlQuery); + file_put_contents($dumpDir . '/' . str_replace('/', '', utf8_decode($doc->_id)) . '.json', json_encode($doc)); + } } mysql_free_result($mysqlQuery);