From bcd6785f73ab79d1a621d7ac19f831ca12ac739a Mon Sep 17 00:00:00 2001 From: mad_dev Date: Mon, 25 Aug 2014 12:05:54 +0300 Subject: [PATCH 1/9] Added config file and DB creation script --- conf/info.conf | 5 +++ db.py | 107 +++++++++++++++++++++++++++++++++++++++++++++++++ masterBot.py | 28 ++++++++++++- 3 files changed, 138 insertions(+), 2 deletions(-) create mode 100644 conf/info.conf create mode 100644 db.py diff --git a/conf/info.conf b/conf/info.conf new file mode 100644 index 0000000..c7812cf --- /dev/null +++ b/conf/info.conf @@ -0,0 +1,5 @@ +[SYS] +host= +user= +pass= +db=fbotDB \ No newline at end of file diff --git a/db.py b/db.py new file mode 100644 index 0000000..010e15d --- /dev/null +++ b/db.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +''' +Copyright (c) 2014 PuN1sh3r luiguibiker@gmail.com + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +''' + +import MySQLdb +import ConfigParser + + + +class sql: + + def __init__(self, host, user, pwd, dbname): + self.host = host + self.user = user + self.pwd = pwd + self.dbname = dbname + + def create_database(self): + db = MySQLdb.connect(host=self.host, user=self.user, passwd=self.pwd) + cur = db.cursor() + sql_create = 'CREATE DATABASE %s' %self.dbname + sql_use = 'USE %s' %self.dbname + cur.execute(sql_create) + cur.execute(sql_use) + cur.execute('''CREATE TABLE fbids ( + fbids BIGINT(20), + sent TEXT, + crawled VARCHAR(20) + )''' + ) + + cur.execute('''CREATE TABLE checkins ( + checkin_id CHAR(32), + fbid BIGINT(20), + message TEXT, + coords TEXT, + timestamp TIMESTAMP + )''' + ) + + cur.execute('''CREATE TABLE users ( + fbid BIGINT(20), + profile_url TEXT, + name TEXT, + contact_email TEXT, + interests TEXT, + birthday VARCHAR(20), + hometown_location TEXT, + relationship_status TEXT, + significant_other_id TEXT, + education TEXT, + work VARCHAR(100), + sex TEXT, + devices VARCHAR(100) + )''' + + ) + + cur.execute('''CREATE TABLE family ( + fbid BIGINT(20), + relative_id CHAR(32), + name VARCHAR(80), + relationship VARCHAR(100) + )''' + ) + + db.commit() + cur.close() + db.close() + + +class Conf: + + def __init__(self): + pass + + def getConf(self): + configData = {} + config = ConfigParser.ConfigParser() + config.read('conf/info.conf') + options = config.options('SYS') + for i in options: + configData[i] = config.get('SYS', i) + + return configData + diff --git a/masterBot.py b/masterBot.py index 1947cff..d77dbdd 100644 --- a/masterBot.py +++ b/masterBot.py @@ -35,6 +35,8 @@ import time import hashlib import datetime +import sys +import db categories = ["Autos","Music","Travel","Animals","Sports","Comedy","People","Entertainment","News","Howto","Education","Tech","Nonprofit","Movies"] @@ -50,12 +52,34 @@ vLink = vLink.replace("&feature=youtube_gdata_player","") yt_videos.append(vLink) +config_file = db.Conf() +param = config_file.getConf() +host = param['host'] +user = param['user'] +password = param['pass'] +dbname = param['db'] + try: - dbconn = MySQLdb.connect("127.0.0.1","user","pass","DBname" ) + dbconn = MySQLdb.connect(host, user, password, dbname ) cursor = dbconn.cursor() except Exception as ex: - print ex + + print 'Could not connect to db: %s' + db_create = db.sql(host, user, password, dbname) + + try: + print 'Creating database' + db_create.create_database() + print 'Created %s' %dbname + + except Exception as e: + '''Should add proper error handling as to pinpoint the exception''' + code = re.findall(r'\d+', str(e)) + if '1007' in code: + print 'Database already exists' + else: + print "Please check your credentials" def addFbids(fbids): print "processing received fbid's....." From f2ef60ccdec2b542cca59d6dce81dc02974f401f Mon Sep 17 00:00:00 2001 From: mad_dev Date: Mon, 25 Aug 2014 12:12:56 +0300 Subject: [PATCH 2/9] ALT* README to represent new features; db creation --- README.md | 9 +++++++-- db.py | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1cf739c..eb5ffcc 100644 --- a/README.md +++ b/README.md @@ -37,21 +37,26 @@ facebot is divided by two components. the masterbot.py script is the one that de ## how to use this tool: +Open `conf/info.conf` and fill in your info; you can change `fbotDB` as the database name. + Before you install req*, do ``` sudo apt-get install build-essential python-dev libmysqlclient-dev ``` -The latter will is needed by MySQL-Python, +The latter is needed by MySQL-Python, ``` pip install -r requirements.txt ``` + --- -'Warning: before executing masterbot you need to create the DB in order to store all the loot. the database squema in jpg form is shown on the repo'. +~~'Warning: before executing masterbot you need to create the DB in order to store all the loot. the database squema in jpg form is shown on the repo'.~~ + +`masterbot.py`will create the db for, given `conf/info.conf` is correct. 'warning: in order to leverage "target_mode" a fake facebook app needs to be created and an access token be created.' these steps were done manually due to lack of dev time but thats on the works... diff --git a/db.py b/db.py index 010e15d..691dddd 100644 --- a/db.py +++ b/db.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- ''' -Copyright (c) 2014 PuN1sh3r luiguibiker@gmail.com +Copyright (c) 2014 mad_dev, mad_dev@linuxmail.org Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal From f96fad3fe30676d7dab8dc23acaee9411bed05c2 Mon Sep 17 00:00:00 2001 From: mad_dev Date: Mon, 25 Aug 2014 12:15:47 +0300 Subject: [PATCH 3/9] TYPO --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index eb5ffcc..026f271 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ facebot is divided by two components. the masterbot.py script is the one that de ## how to use this tool: -Open `conf/info.conf` and fill in your info; you can change `fbotDB` as the database name. +Open `conf/info.conf` and fill in your info; you can change `fbotDB`. Before you install req*, do @@ -56,7 +56,7 @@ pip install -r requirements.txt ~~'Warning: before executing masterbot you need to create the DB in order to store all the loot. the database squema in jpg form is shown on the repo'.~~ -`masterbot.py`will create the db for, given `conf/info.conf` is correct. +`masterbot.py`will create the db for you, given `conf/info.conf` is correct. 'warning: in order to leverage "target_mode" a fake facebook app needs to be created and an access token be created.' these steps were done manually due to lack of dev time but thats on the works... From 1f00d778945b914b5883d9dddd93b57fd578256a Mon Sep 17 00:00:00 2001 From: mad_dev Date: Mon, 25 Aug 2014 16:59:46 +0300 Subject: [PATCH 4/9] fixed line 68 --- masterBot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/masterBot.py b/masterBot.py index d77dbdd..2c0b0d1 100644 --- a/masterBot.py +++ b/masterBot.py @@ -65,7 +65,7 @@ except Exception as ex: - print 'Could not connect to db: %s' + print 'Could not connect to db: %s' dbname db_create = db.sql(host, user, password, dbname) try: From 135037b9d5649088823c14cc880ef0f2329ac2dc Mon Sep 17 00:00:00 2001 From: mad_dev Date: Tue, 26 Aug 2014 11:24:14 +0300 Subject: [PATCH 5/9] dbname --- masterBot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/masterBot.py b/masterBot.py index 2c0b0d1..48bda91 100644 --- a/masterBot.py +++ b/masterBot.py @@ -65,7 +65,7 @@ except Exception as ex: - print 'Could not connect to db: %s' dbname + print 'Could not connect to db: %s' %dbname db_create = db.sql(host, user, password, dbname) try: From 92c00708386d6ce6c68bfc455d4ec9919d683219 Mon Sep 17 00:00:00 2001 From: mad_dev Date: Tue, 26 Aug 2014 12:55:43 +0300 Subject: [PATCH 6/9] Added inst* for target_mode --- README.md | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 026f271..296cf36 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ pip install -r requirements.txt ~~'Warning: before executing masterbot you need to create the DB in order to store all the loot. the database squema in jpg form is shown on the repo'.~~ -`masterbot.py`will create the db for you, given `conf/info.conf` is correct. +`masterbot.py` will create the db for you, given `conf/info.conf` is correct. 'warning: in order to leverage "target_mode" a fake facebook app needs to be created and an access token be created.' these steps were done manually due to lack of dev time but thats on the works... @@ -91,5 +91,18 @@ current_city: hometown: "facebook id of the city as per facebook" - + + +#Instructions for getting target_mode to work + +Until the process is automated, please use the following: + +1. Login to https://developers.facebook.com/ and create an app +2. Go to or `GET` +``` +https://graph.facebook.com/oauth/access_token?client_id=YOUR APP ID&client_secret=YOUR APP SECRET&grant_type=client_credentials``` +3. Got to https://developers.facebook.com/tools/explorer and place the `access_token` from the latter request to where it says `Access Token`. Click on `Get Access Token` +4. Copy the `Access Token` and paste it in your config file; `api_key:` + + any questions or suggestions on how to improve this tool i can be contacted at luiguibiker@gmail.com please use the subject [facebot] so i can better track the email. From 824444844d1afa2db6ac5cc1c829acdc339f9773 Mon Sep 17 00:00:00 2001 From: mad_dev Date: Tue, 26 Aug 2014 12:57:05 +0300 Subject: [PATCH 7/9] Added inst* for target_mode#2 --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 296cf36..55b1acb 100644 --- a/README.md +++ b/README.md @@ -98,9 +98,7 @@ hometown: Until the process is automated, please use the following: 1. Login to https://developers.facebook.com/ and create an app -2. Go to or `GET` -``` -https://graph.facebook.com/oauth/access_token?client_id=YOUR APP ID&client_secret=YOUR APP SECRET&grant_type=client_credentials``` +2. Go to or `GET` https://graph.facebook.com/oauth/access_token?client_id=YOUR APP ID&client_secret=YOUR APP SECRET&grant_type=client_credentials 3. Got to https://developers.facebook.com/tools/explorer and place the `access_token` from the latter request to where it says `Access Token`. Click on `Get Access Token` 4. Copy the `Access Token` and paste it in your config file; `api_key:` From df5673d972c9f0ba7f927a744df023c0064e6222 Mon Sep 17 00:00:00 2001 From: mad_dev Date: Tue, 26 Aug 2014 14:05:08 +0300 Subject: [PATCH 8/9] minor fixes to null return --- masterBot.py | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/masterBot.py b/masterBot.py index 48bda91..db4cc64 100644 --- a/masterBot.py +++ b/masterBot.py @@ -72,6 +72,10 @@ print 'Creating database' db_create.create_database() print 'Created %s' %dbname + dbconn = MySQLdb.connect(host, user, password, dbname ) + cursor = dbconn.cursor() + + '''^ Needed for first time launch''' except Exception as e: '''Should add proper error handling as to pinpoint the exception''' @@ -156,6 +160,7 @@ def execute_sql(sql): except Exception as ex: print ex + def process_loot(cargo): fbid = '' @@ -166,15 +171,35 @@ def process_loot(cargo): if k1 == 'about_me': print "saving about me section on database... " work=interests = hometown_location = relationship_status= name=devices=sex=significant_other_id= birthday = contact_email=education=profile_url ='' + columns = ','.join(v1[0].keys()) interests = v1[0]['interests'] - hometown_location = v1[0]['hometown_location']['name'] + #hometown_location = v1[0]['hometown_location']['name'] + ''' + hometown_location fails 'NoneType' object has no attribute '__getitem__' if "hometown_location": null. rm ['name'] fixes it. + Confirmation needed + ''' + try: + hometown_location = v1[0]['hometown_location']['name'] + #print hometown_location + except: + #print 'Hometown not found' + pass + relationship_status = v1[0]['relationship_status'] name = v1[0]['name'] #devices = v1[0]['devices'][0] - devices = v1[0]['devices'][0]['os'] - - print devices + #devices = v1[0]['devices'][0]['os'] + ''' + devices fails 'NoneType' object has no attribute '__getitem__' if "devices": null. + ''' + try: + devices = v1[0]['devices'][0]['os'] + #print devices + except: + #print 'No devices found' + pass + sex = v1[0]['sex'] significant_other_id = v1[0]['significant_other_id'] if v1[0]['work']: From 7928271e6f57a3a73075a2a285d0401f404f2c76 Mon Sep 17 00:00:00 2001 From: mad_dev Date: Wed, 27 Aug 2014 10:00:01 +0300 Subject: [PATCH 9/9] column name fix --- db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db.py b/db.py index 691dddd..4448d72 100644 --- a/db.py +++ b/db.py @@ -44,7 +44,7 @@ def create_database(self): cur.execute(sql_create) cur.execute(sql_use) cur.execute('''CREATE TABLE fbids ( - fbids BIGINT(20), + fbid BIGINT(20), sent TEXT, crawled VARCHAR(20) )'''