diff --git a/README.md b/README.md index c3d7946..7b7b070 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,8 @@ dbName | `String` | `false` | Optionally insert `USE dbName;` a dropTable | `Boolean` | `false` | Optionally insert `DROP TABLE IF EXISTS tableName;` at beginning of `.sql` file seperator | `String` | `','` | Optionally specify `.csv` file field seperator lineSeperator | `String` | `'\n'` | Optionally specify `.csv` file EOL seperator +isEachLineInsert | `Boolean` | `false` | Optionally insert `insert into xxx () values ` for each line,This is especially useful when dealing with large amounts of data +eachLineHandler | `Function` | `undefined` | before write to steam,you can use this function to deal the line string,example process geometry data `everyLineHandler = (line)=>{return line.replace(/(\"POINT\(.+? .+?\)\")/g, "ST_GeomFromText($1)")}` Testing ------- diff --git a/index.js b/index.js index f55a10d..ebe3ae9 100644 --- a/index.js +++ b/index.js @@ -1,7 +1,7 @@ /* requires */ -var util = require('util'); -var Transform = require('stream').Transform; +var util = require("util"); +var Transform = require("stream").Transform; /* constructor */ @@ -11,23 +11,29 @@ function CSV2SQL(options) { return new CSV2SQL(options); } - this.internalBuffer = ''; + this.internalBuffer = ""; this.isFirstDataRow = true; this.isFirstRowColumnNames = true; this.isFirstChunk = true; - this.tableName = options.tableName || 'undefined'; + this.tableName = options.tableName || "undefined"; this.dbName = options.dbName || false; this.dropTable = options.dropTable || false; - this.seperator = options.seperator || ','; - this.lineSeperator = options.lineSeperator || '\n'; + this.seperator = options.seperator || ","; + this.lineSeperator = options.lineSeperator || "\n"; - //helper functions + // every line insert sql + this.isEachLineInsert = options.isEachLineInsert || false; + this.eachLineInsertHead = ""; + + this.eachLineHandler = options.eachLineHandler || null; + + // helper functions this.insertColumnNames = insertColumnNames; this.lineToInsert = lineToInsert; - //init Transform, call super constructor + // init Transform, call super constructor Transform.call(this, options); } util.inherits(CSV2SQL, Transform); @@ -35,18 +41,18 @@ util.inherits(CSV2SQL, Transform); /* implement transform stream */ //TODO: encoding not 'sticking' -CSV2SQL.prototype._transform = function(chunk, enc, cb) { +CSV2SQL.prototype._transform = function (chunk, enc, cb) { this.internalBuffer += chunk.toString(); var newLinePos; var line; var linePush; if (this.isFirstChunk && this.dbName !== false) { - this.push('USE ' + this.dbName + ';\n'); + this.push("USE " + this.dbName + ";\n"); } if (this.isFirstChunk && this.dropTable !== false) { - this.push('DROP TABLE IF EXISTS ' + this.tableName + ';\n'); + this.push("DROP TABLE IF EXISTS " + this.tableName + ";\n"); } if (this.isFirstChunk) { @@ -60,14 +66,19 @@ CSV2SQL.prototype._transform = function(chunk, enc, cb) { this.internalBuffer = this.internalBuffer.substring(newLinePos + 1); if (this.isFirstRowColumnNames) { - linePush = this.insertColumnNames(line); + if (this.isEachLineInsert) { + this.eachLineInsertHead = this.insertColumnNames(line); + } else { + linePush = this.insertColumnNames(line); + this.push(linePush + "\n"); + } } else { linePush = this.lineToInsert(line); + this.push(linePush + "\n"); } + newLinePos = this.internalBuffer.indexOf(this.lineSeperator); - - this.push(linePush + '\n'); } cb(); @@ -76,8 +87,8 @@ CSV2SQL.prototype._transform = function(chunk, enc, cb) { /* implement transform flush 'event' */ //after all the chunks have been processed, put a ';' to finish off the INSERT -CSV2SQL.prototype._flush = function(cb) { - this.push(';'); +CSV2SQL.prototype._flush = function (cb) { + this.push(";"); cb(); }; @@ -89,16 +100,16 @@ module.exports = CSV2SQL; function insertColumnNames(line) { var columnNamesArr = line.split(this.seperator); - var columnNames = '('; + var columnNames = "("; for (var i = 0; i < columnNamesArr.length; i++) { - columnNames += columnNamesArr[i] + ','; + columnNames += columnNamesArr[i] + ","; } //remove trailing comma columnNames = columnNames.substring(0, columnNames.length - 1); - columnNames += ')'; + columnNames += ")"; - var insert = 'INSERT INTO ' + this.tableName + ' ' + columnNames + ' ' + - 'VALUES'; + var insert = + "INSERT INTO " + this.tableName + " " + columnNames + " " + "VALUES"; this.isFirstRowColumnNames = false; @@ -110,20 +121,22 @@ function insertColumnNames(line) { function lineToInsert(line) { //TODO: use a csv parser here, or write own var dataArr = line.split(this.seperator); - var row; - + var row = ""; + if (this.isEachLineInsert && this.eachLineInsertHead) { + row = this.eachLineInsertHead; + } //insert comma's between VALUES (..), (..), ... , (..) if (this.isFirstDataRow) { - row = '('; + row += "("; this.isFirstDataRow = false; } else { - row = ',('; + row += this.isEachLineInsert ? " (" : ",("; } //build up the row (a, b, ... , c) for (var i = 0; i < dataArr.length; i++) { - if (dataArr[i] === '' || dataArr[i] === 'NULL') { - row += 'NULL'; + if (dataArr[i] === "" || dataArr[i] === "NULL") { + row += "NULL"; } else { //enclose datums in quotes row += '"' + dataArr[i] + '"'; @@ -131,10 +144,12 @@ function lineToInsert(line) { //insert comma's between datums if (i !== dataArr.length - 1) { - row += ','; + row += ","; } } - row += ')'; - + row += this.isEachLineInsert ? ");" : ")"; + if (this.eachLineHandler) { + row = this.eachLineHandler(row); + } return row; } diff --git a/package.json b/package.json index b36c2b3..6fb3136 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,6 @@ }, "homepage": "https://github.com/jub3i/csv2sql-lite", "devDependencies": { - "async": "^0.9.0" + "async": "^0.9.2" } } diff --git a/test/index.js b/test/index.js index 8fc9bed..48029d2 100644 --- a/test/index.js +++ b/test/index.js @@ -2,7 +2,8 @@ var tests = [ fixtureTest, - dropTableTest + dropTableTest, + eachLineTest ]; var async = require('async'); @@ -118,3 +119,44 @@ function dropTableTest(cb) { }); }); } + +function eachLineTest(cb) { + var CSV2SQL = require('../index.js'); + var csv2sql = CSV2SQL({ + tableName: 'tableNameAmazing', + dbName: 'dbNameSpectacular', + isEachLineInsert: true, + }); + var fs = require('fs'); + var rstream; + var sql = ''; + + rstream = fs.createReadStream('./test/fixture.csv'); + + rstream.on('error', function(e) { + cb(e); + }); + + rstream.on('open', function() { + rstream.pipe(csv2sql); + + csv2sql.on('data', function(chunk) { + sql += chunk; + }); + + csv2sql.on('end', function() { + console.log(sql) + var expectedSql = +`USE dbNameSpectacular; +INSERT INTO tableNameAmazing (deviceId,date,latitude,longitude,altitude,accuracy,dataType,ctxFolder,capture) VALUES("Elephant","2014/07/28 13:24:39","-25.821133","28.15825","1428","0","2","yoshi@email.org",NULL); +INSERT INTO tableNameAmazing (deviceId,date,latitude,longitude,altitude,accuracy,dataType,ctxFolder,capture) VALUES ("Elephant","2014/07/28 13:32:49","-25.821117","28.15825","1430","0","2","yoshi@email.org",NULL); +INSERT INTO tableNameAmazing (deviceId,date,latitude,longitude,altitude,accuracy,dataType,ctxFolder,capture) VALUES ("Elephant","2014/07/28 13:40:11","-25.821967","28.16035","1424","0",NULL,"yoshi@email.org",NULL); +;`; + if (sql === expectedSql) { + cb(null, 'generated expected sql'); + } else { + cb('did not produce expected sql'); + } + }); + }); +} \ No newline at end of file