add isEachLineInsert & eachLineHandler support

README.md

@@ -111,6 +111,8 @@ dbName        | `String`     | `false`       | Optionally insert `USE dbName;` a
 dropTable     | `Boolean`    | `false`       | Optionally insert `DROP TABLE IF EXISTS tableName;` at beginning of `.sql` file
 seperator     | `String`     | `','`         | Optionally specify `.csv` file field seperator
 lineSeperator | `String`     | `'\n'`        | Optionally specify `.csv` file EOL seperator
+isEachLineInsert | `Boolean`    | `false`       | Optionally insert `insert into xxx () values ` for each line,This is especially useful when dealing with large amounts of data
+eachLineHandler | `Function`     | `undefined`        | before write to steam,you can use this function to deal the line string,example process geometry data `everyLineHandler = (line)=>{return line.replace(/(\"POINT\(.+? .+?\)\")/g, "ST_GeomFromText($1)")}`
 
 Testing
 -------

index.js

@@ -1,7 +1,7 @@
 /* requires */
 
-var util = require('util');
-var Transform = require('stream').Transform;
+var util = require("util");
+var Transform = require("stream").Transform;
 
 /* constructor */
 
@@ -11,42 +11,48 @@ function CSV2SQL(options) {
     return new CSV2SQL(options);
   }
 
-  this.internalBuffer = '';
+  this.internalBuffer = "";
 
   this.isFirstDataRow = true;
   this.isFirstRowColumnNames = true;
   this.isFirstChunk = true;
 
-  this.tableName = options.tableName || 'undefined';
+  this.tableName = options.tableName || "undefined";
   this.dbName = options.dbName || false;
   this.dropTable = options.dropTable || false;
-  this.seperator = options.seperator || ',';
-  this.lineSeperator = options.lineSeperator || '\n';
+  this.seperator = options.seperator || ",";
+  this.lineSeperator = options.lineSeperator || "\n";
 
-  //helper functions
+  // every line insert sql
+  this.isEachLineInsert = options.isEachLineInsert || false;
+  this.eachLineInsertHead = "";
+
+  this.eachLineHandler = options.eachLineHandler || null;
+
+  // helper functions
   this.insertColumnNames = insertColumnNames;
   this.lineToInsert = lineToInsert;
 
-  //init Transform, call super constructor
+  // init Transform, call super constructor
   Transform.call(this, options);
 }
 util.inherits(CSV2SQL, Transform);
 
 /* implement transform stream */
 
 //TODO: encoding not 'sticking'
-CSV2SQL.prototype._transform = function(chunk, enc, cb) {
+CSV2SQL.prototype._transform = function (chunk, enc, cb) {
   this.internalBuffer += chunk.toString();
   var newLinePos;
   var line;
   var linePush;
 
   if (this.isFirstChunk && this.dbName !== false) {
-    this.push('USE ' + this.dbName + ';\n');
+    this.push("USE " + this.dbName + ";\n");
   }
 
   if (this.isFirstChunk && this.dropTable !== false) {
-    this.push('DROP TABLE IF EXISTS ' + this.tableName + ';\n');
+    this.push("DROP TABLE IF EXISTS " + this.tableName + ";\n");
   }
 
   if (this.isFirstChunk) {
@@ -60,14 +66,19 @@ CSV2SQL.prototype._transform = function(chunk, enc, cb) {
     this.internalBuffer = this.internalBuffer.substring(newLinePos + 1);
 
     if (this.isFirstRowColumnNames) {
-      linePush = this.insertColumnNames(line);
+      if (this.isEachLineInsert) {
+        this.eachLineInsertHead = this.insertColumnNames(line);
+      } else {
+        linePush = this.insertColumnNames(line);
+        this.push(linePush + "\n");
+      }
     } else {
       linePush = this.lineToInsert(line);
+      this.push(linePush + "\n");
     }
+
 
     newLinePos = this.internalBuffer.indexOf(this.lineSeperator);
-
-    this.push(linePush + '\n');
   }
 
   cb();
@@ -76,8 +87,8 @@ CSV2SQL.prototype._transform = function(chunk, enc, cb) {
 /* implement transform flush 'event' */
 
 //after all the chunks have been processed, put a ';' to finish off the INSERT
-CSV2SQL.prototype._flush = function(cb) {
-  this.push(';');
+CSV2SQL.prototype._flush = function (cb) {
+  this.push(";");
   cb();
 };
 
@@ -89,16 +100,16 @@ module.exports = CSV2SQL;
 
 function insertColumnNames(line) {
   var columnNamesArr = line.split(this.seperator);
-  var columnNames = '(';
+  var columnNames = "(";
   for (var i = 0; i < columnNamesArr.length; i++) {
-    columnNames += columnNamesArr[i] + ',';
+    columnNames += columnNamesArr[i] + ",";
   }
   //remove trailing comma
   columnNames = columnNames.substring(0, columnNames.length - 1);
-  columnNames += ')';
+  columnNames += ")";
 
-  var insert = 'INSERT INTO ' + this.tableName + ' ' + columnNames + ' ' +
-    'VALUES';
+  var insert =
+    "INSERT INTO " + this.tableName + " " + columnNames + " " + "VALUES";
 
   this.isFirstRowColumnNames = false;
 
@@ -110,31 +121,35 @@ function insertColumnNames(line) {
 function lineToInsert(line) {
   //TODO: use a csv parser here, or write own
   var dataArr = line.split(this.seperator);
-  var row;
-
+  var row = "";
+  if (this.isEachLineInsert && this.eachLineInsertHead) {
+    row = this.eachLineInsertHead;
+  }
   //insert comma's between VALUES (..), (..), ... , (..)
   if (this.isFirstDataRow) {
-    row = '(';
+    row += "(";
     this.isFirstDataRow = false;
   } else {
-    row = ',(';
+    row += this.isEachLineInsert ? " (" : ",(";
   }
 
   //build up the row (a, b, ... , c)
   for (var i = 0; i < dataArr.length; i++) {
-    if (dataArr[i] === '' || dataArr[i] === 'NULL') {
-      row += 'NULL';
+    if (dataArr[i] === "" || dataArr[i] === "NULL") {
+      row += "NULL";
     } else {
       //enclose datums in quotes
       row += '"' + dataArr[i] + '"';
     }
 
     //insert comma's between datums
     if (i !== dataArr.length - 1) {
-      row += ',';
+      row += ",";
     }
   }
-  row += ')';
-
+  row += this.isEachLineInsert ? ");" : ")";
+  if (this.eachLineHandler) {
+    row = this.eachLineHandler(row);
+  }
   return row;
 }

package.json

@@ -26,6 +26,6 @@
   },
   "homepage": "https://github.com/jub3i/csv2sql-lite",
   "devDependencies": {
-    "async": "^0.9.0"
+    "async": "^0.9.2"
   }
 }

test/index.js

@@ -2,7 +2,8 @@
 
 var tests = [
   fixtureTest,
-  dropTableTest
+  dropTableTest,
+  eachLineTest
 ];
 
 var async = require('async');
@@ -118,3 +119,44 @@ function dropTableTest(cb) {
     });
   });
 }
+
+function eachLineTest(cb) {
+  var CSV2SQL = require('../index.js');
+  var csv2sql = CSV2SQL({
+    tableName: 'tableNameAmazing',
+    dbName: 'dbNameSpectacular',
+    isEachLineInsert: true,
+  });
+  var fs = require('fs');
+  var rstream;
+  var sql = '';
+
+  rstream = fs.createReadStream('./test/fixture.csv');
+
+  rstream.on('error', function(e) {
+    cb(e);
+  });
+
+  rstream.on('open', function() {
+    rstream.pipe(csv2sql);
+
+    csv2sql.on('data', function(chunk) {
+      sql += chunk;
+    });
+
+    csv2sql.on('end', function() {
+      console.log(sql)
+      var expectedSql =
+`USE dbNameSpectacular;
+INSERT INTO tableNameAmazing (deviceId,date,latitude,longitude,altitude,accuracy,dataType,ctxFolder,capture) VALUES("Elephant","2014/07/28 13:24:39","-25.821133","28.15825","1428","0","2","yoshi@email.org",NULL);
+INSERT INTO tableNameAmazing (deviceId,date,latitude,longitude,altitude,accuracy,dataType,ctxFolder,capture) VALUES ("Elephant","2014/07/28 13:32:49","-25.821117","28.15825","1430","0","2","yoshi@email.org",NULL);
+INSERT INTO tableNameAmazing (deviceId,date,latitude,longitude,altitude,accuracy,dataType,ctxFolder,capture) VALUES ("Elephant","2014/07/28 13:40:11","-25.821967","28.16035","1424","0",NULL,"yoshi@email.org",NULL);
+;`;
+      if (sql === expectedSql) {
+        cb(null, 'generated expected sql');
+      } else {
+        cb('did not produce expected sql');
+      }
+    });
+  });
+}