Dump to filesystem

sql2couchdb.php

@@ -61,13 +61,16 @@ function castSQLToPHP($mysqlType, $value)
     case 'bool':        $value = (bool) $value; break;
     case 'boolean':     $value = (bool) $value; break;
     case 'tinyint':     $value = (bool) $value; break;
+    default:            $value = utf8_encode($value); break;
   }
 
   return $value;
 }
 
 function map($row, $json, $mysqlQuery)
 {
+  $wasObject = false;
+
   if($json)
   {
     //turn objects into associative arrays
@@ -81,7 +84,14 @@ function map($row, $json, $mysqlQuery)
     {
       foreach($json as $k => $v)
       {
-        $json[$k] = map($row, $v, $mysqlQuery);
+        $v = map($row, $v, $mysqlQuery);
+        if ($v === null)
+        {
+          unset($json[$k]);
+          continue;
+        } else {
+          $json[$k] = $v;
+        }
 
         if($k == '_id')
           $json[$k] = (string) $json[$k];
@@ -90,6 +100,8 @@ function map($row, $json, $mysqlQuery)
     elseif(is_string($json))
       if($row[$json])
         return castSQLToPHP(getColType($json, $mysqlQuery), $row[$json]);
+      else
+        return null;
 
     if($wasObject)
       $json = (object) $json;
@@ -111,7 +123,8 @@ function map($row, $json, $mysqlQuery)
                     "couchdb-pass:",
                     "couchdb-db:",
                     "couchdb-host:",
-                    "couchdb-port:"
+                    "couchdb-port:",
+                    "dump-here:"
                   )
 );
 
@@ -125,6 +138,8 @@ function map($row, $json, $mysqlQuery)
 $couchdbConfig->host = '127.0.0.1';
 $couchdbConfig->port = '5984';
 
+$dumpDir = false;
+
 foreach($options as $opt => $value)
 {
   switch($opt)
@@ -168,6 +183,10 @@ function map($row, $json, $mysqlQuery)
     case 'couchdb-port':
       $couchdbConfig->port = $value;
       break;
+
+    case 'dump-here':
+      $dumpDir = $value;
+      break;
   }
 }
 
@@ -184,16 +203,18 @@ function map($row, $json, $mysqlQuery)
 if(!$jsonFile->doc || !is_object($jsonFile->doc))
   throw new Exception('Missing the doc.');
 
-// Set up Sag now so that we don't run the whole MySQL loop and then end up
-// with an error.
+if(!$dumpDir) {
+	// Set up Sag now so that we don't run the whole MySQL loop and then end up
+	// with an error.
 
-$sag = new Sag($couchdbConfig->host, $couchdbConfig->port);
-$sag->setDatabase($couchdbConfig->db);
+	$sag = new Sag($couchdbConfig->host, $couchdbConfig->port);
+	$sag->setDatabase($couchdbConfig->db);
 
-if($couchdbConfig->user || $couchdbConfig->pass)
-  $sag->login($couchdbConfig->user, $couchdbConfig->pass);
+	if($couchdbConfig->user || $couchdbConfig->pass)
+	  $sag->login($couchdbConfig->user, $couchdbConfig->pass);
+}
 
-// Get data from MySQL and send it to CouchDB.
+// Get data from MySQL and send it to CouchDB or the $dumpDir
 
 if(!($mysqlCon = mysql_connect("{$mysqlConfig->host}:{$mysqlConfig->port}", $mysqlConfig->user, $mysqlConfig->pass)))
   throw new Exception('Unable to connect to MySQL: '.mysql_error());
@@ -204,12 +225,31 @@ function map($row, $json, $mysqlQuery)
 if(!($mysqlQuery = mysql_query($jsonFile->query, $mysqlCon)))
   throw new Exception('Invalid query: '.mysql_error());
 
-$docsToSend = array();
-
-while($row = mysql_fetch_array($mysqlQuery, MYSQL_ASSOC))
-  $docsToSend[] = map($row, clone $jsonFile->doc, $mysqlQuery);
-
-$sag->bulk($docsToSend);
+if(!$dumpDir) {
+	$docsToSend = array();
+
+	$c = 0;
+	while($row = mysql_fetch_array($mysqlQuery, MYSQL_ASSOC))  {
+	  $docsToSend[] = map($row, clone $jsonFile->doc, $mysqlQuery);
+	  $c++;
+	  if ($c === 1001) {
+	    $sag->bulk($docsToSend);
+	    $docsToSend = array();
+	    $c = 0;
+	  }
+	}
+
+	if (count($docsToSend) > 0) {
+	  $sag->bulk($docsToSend);
+	}
+} else {
+	if(!file_exists($dumpDir))
+		mkdir($dumpDir);
+	while($row = mysql_fetch_array($mysqlQuery, MYSQL_ASSOC))  {
+		$doc = map($row, clone $jsonFile->doc, $mysqlQuery);
+		file_put_contents($dumpDir . '/' . str_replace('/', '', utf8_decode($doc->_id)) . '.json', json_encode($doc));
+	}
+}
 
 mysql_free_result($mysqlQuery);
 mysql_close($mysqlCon);