From 922cfdc3356dd93ba8a8a199f954bfcdd8f0c7c5 Mon Sep 17 00:00:00 2001 From: Zack Penka Date: Fri, 1 Apr 2016 19:29:18 -0400 Subject: [PATCH 1/6] Initial Housekeeping Lint files, remove core package from dependencies --- .eslintrc | 3 ++ app.js | 92 +++++++++++++++++++++++++--------------------------- package.json | 5 ++- 3 files changed, 52 insertions(+), 48 deletions(-) create mode 100644 .eslintrc diff --git a/.eslintrc b/.eslintrc new file mode 100644 index 0000000..2669de2 --- /dev/null +++ b/.eslintrc @@ -0,0 +1,3 @@ +{ + "extends": "airbnb/base" +} diff --git a/app.js b/app.js index 72cf089..2026e91 100644 --- a/app.js +++ b/app.js @@ -1,63 +1,63 @@ -const jsdom = require('jsdom'); -const argv = require('yargs').argv; -const fs = require('fs'); -const _ = require('lodash'); -const request = require('request'); -const jsonfile = require('jsonfile'); -const logger = require("./utils/logger"); +import jsdom from 'jsdom'; +import { argv } from 'yargs'; +import fs from 'fs'; +import _ from 'lodash'; +import request from 'request'; +import jsonfile from 'jsonfile'; +import logger from './utils/logger'; let IFTTTParams = {}; -let IFTTTTimers = {}; +const IFTTTTimers = {}; const { endpoints, interval, ifttt } = argv; const options = { selector: 'body :not(script)', jQuerySrc: 'http://code.jquery.com/jquery.js', - defaultTimeout: 10 + defaultTimeout: 10, }; // Ensure we have required arguments -if(!_.isString(endpoints) || !_.isInteger(interval) || interval === 0) { - console.error('--endpoints and --interval are required'); +if (!_.isString(endpoints) || !_.isInteger(interval) || interval === 0) { + logger('--endpoints and --interval are required'); process.exit(1); } // Ensure list of endpoints is a file -if(!fs.statSync(endpoints).isFile()) { - console.error('--endpoints should refer to a file (list of endpoints)'); +if (!fs.statSync(endpoints).isFile()) { + logger('--endpoints should refer to a file (list of endpoints)'); process.exit(2); } // Ensure IFTTT configuration is valid -if(ifttt) { - if(!fs.statSync(ifttt).isFile()) { - console.error('--ifttt should refer to a JSON file configuration'); +if (ifttt) { + if (!fs.statSync(ifttt).isFile()) { + logger('--ifttt should refer to a JSON file configuration'); process.exit(5); } const { key, eventName, bodyKey } = IFTTTParams = jsonfile.readFileSync(ifttt); - if(!key || !eventName || !bodyKey || !_.isString(key) || !_.isString(eventName) || !_.isString(bodyKey)) { - console.error('--ifttt file is missing required data'); + if (!key || !eventName || !bodyKey || !_.isString(key) || !_.isString(eventName) || !_.isString(bodyKey)) { // eslint-disable-line max-len + logger('--ifttt file is missing required data'); process.exit(6); } } // Make requests to endpoints const makeRequests = (urls, callback) => { - let responses = {}; + const responses = {}; let complete = 0; urls.forEach((url) => { jsdom.env({ - url: url, + url, scripts: [options.jQuerySrc], done: (err, window) => { - if(!window || !window.$ || err) { - console.error(`Resource data located at ${url} failed to load`); + if (!window || !window.$ || err) { + logger(`Resource data located at ${url} failed to load`); } else { const $ = window.$; - $(options.selector).each(function() { + $(options.selector).each(() => { const responseText = $(this).text().replace(/\W+/g, ''); responses[url] = responseText; @@ -66,10 +66,10 @@ const makeRequests = (urls, callback) => { complete++; - if(complete === urls.length) { + if (complete === urls.length) { callback(responses); } - } + }, }); }); }; @@ -80,40 +80,38 @@ const postIFTTT = (data) => { const timeout = (_.isInteger(IFTTTParams.timeout) ? IFTTTParams.timeout : options.defaultTimeout); // Ensure enough time has passed since last time an event was dispatched - if(!IFTTTTimers[data] || now - IFTTTTimers[data] > timeout) { - let postData = {}; + if (!IFTTTTimers[data] || now - IFTTTTimers[data] > timeout) { + const postData = {}; postData[IFTTTParams.bodyKey] = data; IFTTTTimers[data] = now; request.post({ url: `https://maker.ifttt.com/trigger/${IFTTTParams.eventName}/with/key/${IFTTTParams.key}`, - form: postData + form: postData, }, (err, response) => { - if(err) { - console.log('- IFTTT event dispatch failed'); + if (err) { + logger('- IFTTT event dispatch failed'); } else { - console.log('- IFTTT event dispatched'); + logger('- IFTTT event dispatched', response); } }); } else { - console.log('- IFTTT event ignored due to timeout'); + logger('- IFTTT event ignored due to timeout'); } }; // Read endpoints file and create list of endpoints fs.readFile(endpoints, 'utf-8', (err, data) => { - if(err) { - console.error('--endpoints file could not be read'); + if (err) { + logger('--endpoints file could not be read'); process.exit(3); } - const endpointsList = _.remove(data.split('\n'), (item) => { - return _.isString(item) && !_.isEmpty(item); - }); + const endpointsList = _.remove(data.split('\n'), (item) => _.isString(item) && !_.isEmpty(item)); - if(!endpointsList.length) { - console.error('--endpoints file does not contain any endpoints'); + if (!endpointsList.length) { + logger('--endpoints file does not contain any endpoints'); process.exit(4); } @@ -121,26 +119,26 @@ fs.readFile(endpoints, 'utf-8', (err, data) => { makeRequests(endpointsList, (responses) => { const cache = responses; - console.log(`${_.keys(responses).length} of ${endpointsList.length} responses cached`); + logger(`${_.keys(responses).length} of ${endpointsList.length} responses cached`); setInterval(() => { - makeRequests(endpointsList, (responses) => { - const differences = _.difference(_.values(responses), _.values(cache)); + makeRequests(endpointsList, (requestResponses) => { + const differences = _.difference(_.values(requestResponses), _.values(cache)); - if(differences.length) { + if (differences.length) { differences.forEach((difference) => { - const endpoint = _.invert(responses)[difference]; + const endpoint = _.invert(requestResponses)[difference]; cache[endpoint] = difference; - console.log(`Difference identified within ${endpoint}`); + logger(`Difference identified within ${endpoint}`); - if(ifttt) { + if (ifttt) { postIFTTT(endpoint); } }); } else { - console.log(`No differences identified for ${_.keys(responses).length} responses`) + logger(`No differences identified for ${_.keys(requestResponses).length} responses`); } }); }, interval * 1000); diff --git a/package.json b/package.json index e6e4615..2b9e2ed 100644 --- a/package.json +++ b/package.json @@ -15,12 +15,15 @@ "dependencies": { "babel-preset-es2015": "^6.6.0", "babel-register": "^6.7.2", - "fs": "0.0.2", "jsdom": "^8.1.0", "jsonfile": "^2.2.3", "lodash": "^4.6.1", "request": "^2.69.0", "winston": "^2.2.0", "yargs": "^4.3.2" + }, + "devDependencies": { + "eslint": "^2.6.0", + "eslint-config-airbnb": "^6.2.0" } } From e71fde9b10e5902ea5e5670f6bd5c006f9c61e92 Mon Sep 17 00:00:00 2001 From: Zack Penka Date: Sat, 2 Apr 2016 09:05:36 -0400 Subject: [PATCH 2/6] Refactor to use Promises --- app.js | 219 ++++++++++++++++++++++++---------------------- example/endpoints | 6 +- package.json | 1 + utils/logger.js | 12 +-- 4 files changed, 121 insertions(+), 117 deletions(-) diff --git a/app.js b/app.js index 2026e91..a36023c 100644 --- a/app.js +++ b/app.js @@ -1,146 +1,153 @@ import jsdom from 'jsdom'; -import { argv } from 'yargs'; -import fs from 'fs'; -import _ from 'lodash'; -import request from 'request'; -import jsonfile from 'jsonfile'; +const argv = require('yargs').argv; +import { keys, invert, difference, values } from 'lodash'; import logger from './utils/logger'; +import Promise from 'bluebird'; -let IFTTTParams = {}; const IFTTTTimers = {}; -const { endpoints, interval, ifttt } = argv; +const IFTTTParams = {}; +const endpointsPath = argv.endpoints; +const interval = argv.interval; +const iftttPath = argv.ifttt; const options = { selector: 'body :not(script)', jQuerySrc: 'http://code.jquery.com/jquery.js', defaultTimeout: 10, }; -// Ensure we have required arguments -if (!_.isString(endpoints) || !_.isInteger(interval) || interval === 0) { - logger('--endpoints and --interval are required'); +// Promisify imports +import { + stat as _stat, + readFile as _readFile, +} from 'fs'; +const stat = Promise.promisify(_stat); +const readFile = Promise.promisify(_readFile); + +import { post as _post } from 'request'; +const post = Promise.promisify(_post); + +// Ensure we have valid interval arg +if (!interval || !parseInt(interval, 10) > 0) { + logger.error('--interval is required'); process.exit(1); } -// Ensure list of endpoints is a file -if (!fs.statSync(endpoints).isFile()) { - logger('--endpoints should refer to a file (list of endpoints)'); - process.exit(2); -} +// Ensure endpoints list is a file +stat(endpointsPath) +.catch((err) => { + logger.error('--endpoints must be a valid file and is required', err); + process.exit(1); +}); -// Ensure IFTTT configuration is valid -if (ifttt) { - if (!fs.statSync(ifttt).isFile()) { - logger('--ifttt should refer to a JSON file configuration'); - process.exit(5); +// Read endpoints file and create list of endpoints +const endpoints = readFile(endpointsPath, 'utf-8') +.then((endpointsList) => { + if (!endpointsList.length) { + logger.error('--endpoints file does not contain any endpoints'); + process.exit(4); } - const { key, eventName, bodyKey } = IFTTTParams = jsonfile.readFileSync(ifttt); + return endpointsList.split('\n').filter((endpoint) => endpoint && typeof endpoint === 'string'); +}) +.catch((err) => { + logger.error('--endpoints file could not be read', err); + process.exit(3); +}); - if (!key || !eventName || !bodyKey || !_.isString(key) || !_.isString(eventName) || !_.isString(bodyKey)) { // eslint-disable-line max-len - logger('--ifttt file is missing required data'); - process.exit(6); - } -} +// Create IFTTT Parameters +readFile(iftttPath) + .then((file) => JSON.parse(file)) + .then((params) => { + // Validate IFTTT Parameters + if ( + typeof params.key !== 'string' || + typeof params.eventName !== 'string' || + typeof params.bodyKey !== 'string' + ) { + logger.error('--ifttt file is missing required data'); + process.exit(6); + } else { + IFTTTParams.key = params.key; + IFTTTParams.eventName = params.eventName; + IFTTTParams.bodyKey = params.bodyKey; + IFTTTParams.optionalTimeout = params.timeout; + } + }) + .catch((err) => logger.error('--ifttt should refer to a JSON file configuration', err)); // Make requests to endpoints -const makeRequests = (urls, callback) => { +const makeRequests = (urls) => { const responses = {}; - let complete = 0; - - urls.forEach((url) => { - jsdom.env({ - url, - scripts: [options.jQuerySrc], - done: (err, window) => { - if (!window || !window.$ || err) { - logger(`Resource data located at ${url} failed to load`); - } else { - const $ = window.$; - - $(options.selector).each(() => { - const responseText = $(this).text().replace(/\W+/g, ''); - - responses[url] = responseText; - }); - } - - complete++; - - if (complete === urls.length) { - callback(responses); - } - }, - }); - }); + + Promise.map(urls, (url) => jsdom.env({ + url, + scripts: [options.jQuerySrc], + done: (err, window) => { + if (!window || !window.$ || err) { + logger.error(`Resource data located at ${url} failed to load`); + } else { + const $ = window.$; + $(options.selector).each(() => { + responses[url] = $(this).text().replace(/\W+/g, ''); + }); + } + }, + })); + + return Promise.resolve(responses); }; // Send event to IFTTT const postIFTTT = (data) => { const now = Math.round(new Date().getTime() / 1000); - const timeout = (_.isInteger(IFTTTParams.timeout) ? IFTTTParams.timeout : options.defaultTimeout); + const timeout = (IFTTTParams.optionalTimeout && typeof IFTTTParams.optionalTimeout === 'number') ? + IFTTTParams.optionalTimeout : + options.defaultTimeout; // Ensure enough time has passed since last time an event was dispatched if (!IFTTTTimers[data] || now - IFTTTTimers[data] > timeout) { - const postData = {}; - - postData[IFTTTParams.bodyKey] = data; IFTTTTimers[data] = now; - - request.post({ + const request = { url: `https://maker.ifttt.com/trigger/${IFTTTParams.eventName}/with/key/${IFTTTParams.key}`, - form: postData, - }, (err, response) => { - if (err) { - logger('- IFTTT event dispatch failed'); - } else { - logger('- IFTTT event dispatched', response); - } - }); + form: { bodyKey: data }, + }; + + post(request) + .then((response) => logger.info('- IFTTT event dispatched', response)) + .catch((err) => logger.error('- IFTT event dispatch failed', err)); } else { - logger('- IFTTT event ignored due to timeout'); + logger.info('- IFTTT event ignored due to timeout'); } }; -// Read endpoints file and create list of endpoints -fs.readFile(endpoints, 'utf-8', (err, data) => { - if (err) { - logger('--endpoints file could not be read'); - process.exit(3); - } - - const endpointsList = _.remove(data.split('\n'), (item) => _.isString(item) && !_.isEmpty(item)); - - if (!endpointsList.length) { - logger('--endpoints file does not contain any endpoints'); - process.exit(4); - } - - // Cache initial endpoint responses - makeRequests(endpointsList, (responses) => { - const cache = responses; - - logger(`${_.keys(responses).length} of ${endpointsList.length} responses cached`); - - setInterval(() => { - makeRequests(endpointsList, (requestResponses) => { - const differences = _.difference(_.values(requestResponses), _.values(cache)); - - if (differences.length) { - differences.forEach((difference) => { - const endpoint = _.invert(requestResponses)[difference]; +// Cache passed responses and then setup diffing intervals +const diffCacheInterval = (responses) => { + const cache = responses; + logger.info(`${keys(responses).length} of ${endpoints.length} responses cached.`); - cache[endpoint] = difference; + const poll = () => makeRequests(endpoints).then((pollResponses) => { + const diff = difference(values(pollResponses), values(cache)); - logger(`Difference identified within ${endpoint}`); + if (diff.length) { + diff.forEach((change) => { + const endpoint = invert(pollResponses)[change]; + cache[endpoint] = change; + logger.info(`Difference identified within ${endpoint}`); - if (ifttt) { - postIFTTT(endpoint); - } - }); - } else { - logger(`No differences identified for ${_.keys(requestResponses).length} responses`); - } + postIFTTT(endpoint); }); - }, interval * 1000); + } else { + logger.info(`No differences identified for ${keys(pollResponses).length} responses`); + } }); + + return setInterval(poll, interval * 1000); +}; + +// Start App +makeRequests(endpoints) +.then((responses) => diffCacheInterval(responses)) +.catch((err) => { + logger.error('Error connecting to endpoints', err); + process.exit(1); }); diff --git a/example/endpoints b/example/endpoints index 21366ca..e4616fb 100644 --- a/example/endpoints +++ b/example/endpoints @@ -1,3 +1,3 @@ -http://first.web.site -http://second.web.site -http://third.web.site +http://www.google.com +http://www.facebook.com +http://www.myspace.com diff --git a/package.json b/package.json index 2b9e2ed..d8d6339 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "dependencies": { "babel-preset-es2015": "^6.6.0", "babel-register": "^6.7.2", + "bluebird": "^3.3.4", "jsdom": "^8.1.0", "jsonfile": "^2.2.3", "lodash": "^4.6.1", diff --git a/utils/logger.js b/utils/logger.js index cd49380..6b13be2 100644 --- a/utils/logger.js +++ b/utils/logger.js @@ -3,20 +3,16 @@ const winston = require('winston'); // Define logger configuration -let logger = new winston.Logger({ +const logger = new winston.Logger({ transports: [ new winston.transports.Console({ handleExceptions: true, json: false, colorize: true, - timestamp: true - }) + timestamp: true, + }), ], - exitOnError: false + exitOnError: false, }); -// Use logger in favor of native -console.log = logger.info; -console.error = logger.error; - module.exports = logger; From bb615d04af3a31827863a07adbfb25e4c454ddb0 Mon Sep 17 00:00:00 2001 From: Zack Penka Date: Sat, 2 Apr 2016 09:13:51 -0400 Subject: [PATCH 3/6] Fix bug with unresolved Promise --- app.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/app.js b/app.js index a36023c..a53231b 100644 --- a/app.js +++ b/app.js @@ -6,6 +6,7 @@ import Promise from 'bluebird'; const IFTTTTimers = {}; const IFTTTParams = {}; +let endpoints = {}; const endpointsPath = argv.endpoints; const interval = argv.interval; const iftttPath = argv.ifttt; @@ -40,14 +41,15 @@ stat(endpointsPath) }); // Read endpoints file and create list of endpoints -const endpoints = readFile(endpointsPath, 'utf-8') +const getEndpoints = (path) => readFile(path, 'utf-8') .then((endpointsList) => { if (!endpointsList.length) { logger.error('--endpoints file does not contain any endpoints'); process.exit(4); } - return endpointsList.split('\n').filter((endpoint) => endpoint && typeof endpoint === 'string'); + endpoints = endpointsList.split('\n').filter((endpoint) => endpoint && typeof endpoint === 'string'); //eslint-disable-line + return endpoints; }) .catch((err) => { logger.error('--endpoints file could not be read', err); @@ -145,7 +147,8 @@ const diffCacheInterval = (responses) => { }; // Start App -makeRequests(endpoints) +getEndpoints(endpointsPath) +.then((validEndpoints) => makeRequests(validEndpoints)) .then((responses) => diffCacheInterval(responses)) .catch((err) => { logger.error('Error connecting to endpoints', err); From 9edc3601678b104081e86c5118352ca34e5055f3 Mon Sep 17 00:00:00 2001 From: Zack Penka Date: Sat, 2 Apr 2016 11:15:31 -0400 Subject: [PATCH 4/6] Bring back _ stuff, stronger check on endpointsPath --- app.js | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/app.js b/app.js index a53231b..f53e41e 100644 --- a/app.js +++ b/app.js @@ -1,6 +1,6 @@ import jsdom from 'jsdom'; -const argv = require('yargs').argv; -import { keys, invert, difference, values } from 'lodash'; +import { argv } from 'yargs'; +import _ from 'lodash'; import logger from './utils/logger'; import Promise from 'bluebird'; @@ -28,13 +28,19 @@ import { post as _post } from 'request'; const post = Promise.promisify(_post); // Ensure we have valid interval arg -if (!interval || !parseInt(interval, 10) > 0) { +if (!_.isInteger(interval) || interval < 0) { logger.error('--interval is required'); process.exit(1); } // Ensure endpoints list is a file stat(endpointsPath) +.then((endpointsFile) => { + if (!endpointsFile.isFile()) { + logger.error('--endpoints must be a file'); + process.exit(1); + } +}) .catch((err) => { logger.error('--endpoints must be a valid file and is required', err); process.exit(1); @@ -62,9 +68,9 @@ readFile(iftttPath) .then((params) => { // Validate IFTTT Parameters if ( - typeof params.key !== 'string' || - typeof params.eventName !== 'string' || - typeof params.bodyKey !== 'string' + !_.isString(params.key) || + !_.isString(params.eventName) || + !_.isString(params.bodyKey) ) { logger.error('--ifttt file is missing required data'); process.exit(6); @@ -102,7 +108,7 @@ const makeRequests = (urls) => { // Send event to IFTTT const postIFTTT = (data) => { const now = Math.round(new Date().getTime() / 1000); - const timeout = (IFTTTParams.optionalTimeout && typeof IFTTTParams.optionalTimeout === 'number') ? + const timeout = (IFTTTParams.optionalTimeout && _.isInteger(IFTTTParams.optionalTimeout)) ? IFTTTParams.optionalTimeout : options.defaultTimeout; @@ -125,21 +131,21 @@ const postIFTTT = (data) => { // Cache passed responses and then setup diffing intervals const diffCacheInterval = (responses) => { const cache = responses; - logger.info(`${keys(responses).length} of ${endpoints.length} responses cached.`); + logger.info(`${_.keys(responses).length} of ${endpoints.length} responses cached.`); const poll = () => makeRequests(endpoints).then((pollResponses) => { - const diff = difference(values(pollResponses), values(cache)); + const diff = _.difference(_.values(pollResponses), _.values(cache)); if (diff.length) { diff.forEach((change) => { - const endpoint = invert(pollResponses)[change]; + const endpoint = _.invert(pollResponses)[change]; cache[endpoint] = change; logger.info(`Difference identified within ${endpoint}`); postIFTTT(endpoint); }); } else { - logger.info(`No differences identified for ${keys(pollResponses).length} responses`); + logger.info(`No differences identified for ${_.keys(pollResponses).length} responses`); } }); From c1746da8a2a6f8c25791d0a75d01ad6a34a6ee03 Mon Sep 17 00:00:00 2001 From: Zack Penka Date: Sat, 2 Apr 2016 11:24:02 -0400 Subject: [PATCH 5/6] Fix bug in makeRequests Was not waiting to resolve the responses, meaning it was continuing on before the Promise.map was finished --- app.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/app.js b/app.js index f53e41e..816a077 100644 --- a/app.js +++ b/app.js @@ -100,9 +100,7 @@ const makeRequests = (urls) => { }); } }, - })); - - return Promise.resolve(responses); + })).then(() => Promise.resolve(responses)); }; // Send event to IFTTT From 9aa7c71142f8df6ce5467382c27f8c32651f0ec0 Mon Sep 17 00:00:00 2001 From: Zack Penka Date: Sat, 2 Apr 2016 11:26:04 -0400 Subject: [PATCH 6/6] Error handling on makeRequests --- app.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app.js b/app.js index 816a077..4fa48cb 100644 --- a/app.js +++ b/app.js @@ -100,7 +100,8 @@ const makeRequests = (urls) => { }); } }, - })).then(() => Promise.resolve(responses)); + })).then(() => Promise.resolve(responses)) + .catch((err) => logger.err('There was a problem making requests', err)); }; // Send event to IFTTT