From ca72ee24b37fe6efb2d9477239562aa51332f471 Mon Sep 17 00:00:00 2001 From: josev814 Date: Thu, 28 May 2020 18:05:15 -0400 Subject: [PATCH 1/5] adding ability to receive version report information from clients adding requirements txt file --- denyhosts_server/controllers.py | 38 ++++++++++++++++++++++++++++++++- denyhosts_server/database.py | 26 +++++++++++++++++++++- denyhosts_server/models.py | 14 ++++++++++++ denyhosts_server/views.py | 24 +++++++++++++++++++++ requirements.txt | 13 +++++++++++ 5 files changed, 113 insertions(+), 2 deletions(-) create mode 100644 requirements.txt diff --git a/denyhosts_server/controllers.py b/denyhosts_server/controllers.py index 6998336..75f5fcf 100644 --- a/denyhosts_server/controllers.py +++ b/denyhosts_server/controllers.py @@ -24,7 +24,7 @@ import config import database import models -from models import Cracker, Report, Legacy +from models import Cracker, Report, Legacy, ClientVersion import utils def get_cracker(ip_address): @@ -50,6 +50,42 @@ def handle_report_from_client(client_ip, timestamp, hosts): utils.unlock_host(cracker_ip) logging.debug("Done adding report for {} from {}".format(cracker_ip,client_ip)) +@inlineCallbacks +def handle_version_report_from_client(client_ip, version_info, timestamp): + if not utils.is_valid_ip_address(client_ip): + logging.warning("Illegal remote ip address {}".format(client_ip)) + raise Exception("Illegal remote IP address \"{}\".".format(client_ip)) + + dh_version = version_info[1] + py_version = version_info[0] + try: + client_report = yield ClientVersion.find( + where=['ip_address=? AND denyhosts_version=?', client_ip, dh_version], + limit=1 + ) + if client_report is None: + logging.debug("Adding version report for {}".format(client_ip)) + save_version = ClientVersion( + ip_address=client_ip, + first_time=timestamp, + latest_time=timestamp, + python_version=py_version, + denyhosts_version=dh_version, + total_reports=1 + ) + yield save_version.save() + else: + logging.debug('Updating Client Report: {}'.format(client_report)) + client_report.latest_time = timestamp + client_report.python_version = py_version + client_report.denyhosts_version = dh_version + client_report.total_reports = client_report.total_reports + 1 + yield client_report._update() + + except Exception as e: + logging.exception('Error in Version Reporting: {}'.format(e)) + logging.debug("Done adding report from {}".format(client_ip)) + # Note: lock cracker IP first! # Report merging algorithm by Anne Bezemer, see # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=622697 diff --git a/denyhosts_server/database.py b/denyhosts_server/database.py index 5f58e76..47e6eed 100644 --- a/denyhosts_server/database.py +++ b/denyhosts_server/database.py @@ -38,6 +38,7 @@ def _remove_tables(txn): txn.execute("DROP TABLE IF EXISTS legacy") txn.execute("DROP TABLE IF EXISTS history") txn.execute("DROP TABLE IF EXISTS country_history") + txn.execute("DROP TABLE IF EXISTS ClientVersions") def _evolve_database_initial(txn, dbtype): if dbtype=="sqlite3": @@ -145,6 +146,28 @@ def _evolve_database_v8(txn, dbtype): print("Fixing up historical data...") stats.fixup_history_txn(txn) + +def _evolve_database_v9(txn, dbtype): + global _quiet + if dbtype == "sqlite3": + autoincrement = "AUTOINCREMENT" + elif dbtype == "MySQLdb": + autoincrement = "AUTO_INCREMENT" + + if dbtype == "MySQLdb": + txn.execute("""CREATE TABLE ClientVersions ( + id INTEGER PRIMARY KEY {}, + ip_address VARCHAR(50), + first_time INTEGER, + latest_time INTEGER, + python_version VARCHAR(15), + denyhosts_version VARCHAR(25), + total_reports INTEGER + )""".format(autoincrement)) + txn.execute("CREATE INDEX denyhosts_version_count ON ClientVersions(total_reports)") + txn.execute("CREATE INDEX denyhosts_ip_version ON ClientVersions(denyhosts_version, ip_address)") + + _evolutions = { 1: _evolve_database_v1, 2: _evolve_database_v2, @@ -153,7 +176,8 @@ def _evolve_database_v8(txn, dbtype): 5: _evolve_database_v5, 6: _evolve_database_v6, 7: _evolve_database_v7, - 8: _evolve_database_v8 + 8: _evolve_database_v8, + 9: _evolve_database_v9 } _schema_version = len(_evolutions) diff --git a/denyhosts_server/models.py b/denyhosts_server/models.py index 901ea71..1159f29 100644 --- a/denyhosts_server/models.py +++ b/denyhosts_server/models.py @@ -18,6 +18,18 @@ from twistar.dbobject import DBObject + +class ClientVersion(DBObject): + TABLENAME = 'ClientVersions' + column_names = ['ip_address', 'first_time', 'latest_time', 'python_version', 'denyhosts_version', 'total_reports'] + + def __str__(self): + return "ClientVersion({},{},{},{},{},{},{})".format( + self.id, self.ip_address, self.first_time, self.latest_time, + self.python_version, self.denyhosts_version, self.total_reports + ) + + class Cracker(DBObject): HASMANY=['reports'] column_names=['ip_address','first_time', 'latest_time', 'resiliency', 'total_reports', 'current_reports'] @@ -25,6 +37,7 @@ class Cracker(DBObject): def __str__(self): return "Cracker({},{},{},{},{},{})".format(self.id,self.ip_address,self.first_time,self.latest_time,self.resiliency,self.total_reports,self.current_reports) + class Report(DBObject): BELONGSTO=['cracker'] column_names=['ip_address','first_report_time', 'latest_report_time'] @@ -32,6 +45,7 @@ class Report(DBObject): def __str__(self): return "Report({},{},{},{})".format(self.id,self.ip_address,self.first_report_time,self.latest_report_time) + class Legacy(DBObject): TABLENAME="legacy" pass diff --git a/denyhosts_server/views.py b/denyhosts_server/views.py index cb88582..46c9a8b 100644 --- a/denyhosts_server/views.py +++ b/denyhosts_server/views.py @@ -38,6 +38,30 @@ class Server(xmlrpc.XMLRPC): An example object to be published. """ + @withRequest + @inlineCallbacks + def xmlrpc_version_report(self, request, version_info): + try: + x_real_ip = request.requestHeaders.getRawHeaders("X-Real-IP") + remote_ip = x_real_ip[0] if x_real_ip else request.getClientIP() + now = time.time() + + logging.info("version_report({}) from {}".format(version_info, remote_ip)) + yield controllers.handle_version_report_from_client(remote_ip, version_info, now) + try: + yield peering.send_version_update(remote_ip, version_info, now) + except xmlrpc.Fault, e: + raise e + except Exception, e: + logging.warning("Error sending version report") + except xmlrpc.Fault, e: + raise e + except Exception, e: + log.err(_why="Exception in version_report") + raise xmlrpc.Fault(104, "Error version report: {}".format(e)) + + returnValue(0) + @withRequest @inlineCallbacks def xmlrpc_add_hosts(self, request, hosts): diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..ccc9ea1 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,13 @@ +setuptools ; python_version < '3.0' +twisted +twistar +libnacl +ipaddr ; python_version < '3.0' +GeoIP +jinja2 +matplotlib +numpy +minify +pysqlite ; python_version < '3.0' +mysql-python ; python_version < '3.0' +mysqlclient ; python_version >= '3.0' \ No newline at end of file From de81eb8313c3000f09f36a9472cb0af2a9ea779e Mon Sep 17 00:00:00 2001 From: josev814 Date: Thu, 28 May 2020 18:16:38 -0400 Subject: [PATCH 2/5] removing check for denyhosts_version should only be looking for the ip_address to update the client version for --- denyhosts_server/controllers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/denyhosts_server/controllers.py b/denyhosts_server/controllers.py index 75f5fcf..19fd240 100644 --- a/denyhosts_server/controllers.py +++ b/denyhosts_server/controllers.py @@ -60,7 +60,7 @@ def handle_version_report_from_client(client_ip, version_info, timestamp): py_version = version_info[0] try: client_report = yield ClientVersion.find( - where=['ip_address=? AND denyhosts_version=?', client_ip, dh_version], + where=['ip_address=?', client_ip], limit=1 ) if client_report is None: From 464448454554a116c967a5f136ab6eaa517de46b Mon Sep 17 00:00:00 2001 From: josev814 Date: Fri, 29 May 2020 11:59:52 -0400 Subject: [PATCH 3/5] resolving sqlite table creation resolving table name convention matching --- denyhosts_server/database.py | 10 +++++----- denyhosts_server/models.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/denyhosts_server/database.py b/denyhosts_server/database.py index 47e6eed..16b1284 100644 --- a/denyhosts_server/database.py +++ b/denyhosts_server/database.py @@ -38,7 +38,7 @@ def _remove_tables(txn): txn.execute("DROP TABLE IF EXISTS legacy") txn.execute("DROP TABLE IF EXISTS history") txn.execute("DROP TABLE IF EXISTS country_history") - txn.execute("DROP TABLE IF EXISTS ClientVersions") + txn.execute("DROP TABLE IF EXISTS client_version") def _evolve_database_initial(txn, dbtype): if dbtype=="sqlite3": @@ -154,8 +154,8 @@ def _evolve_database_v9(txn, dbtype): elif dbtype == "MySQLdb": autoincrement = "AUTO_INCREMENT" - if dbtype == "MySQLdb": - txn.execute("""CREATE TABLE ClientVersions ( + if dbtype in ['MySQLdb', 'sqlite3']: + txn.execute("""CREATE TABLE client_version ( id INTEGER PRIMARY KEY {}, ip_address VARCHAR(50), first_time INTEGER, @@ -164,8 +164,8 @@ def _evolve_database_v9(txn, dbtype): denyhosts_version VARCHAR(25), total_reports INTEGER )""".format(autoincrement)) - txn.execute("CREATE INDEX denyhosts_version_count ON ClientVersions(total_reports)") - txn.execute("CREATE INDEX denyhosts_ip_version ON ClientVersions(denyhosts_version, ip_address)") + txn.execute("CREATE INDEX denyhosts_version_count ON client_version(total_reports)") + txn.execute("CREATE INDEX denyhosts_ip_version ON client_version(denyhosts_version, ip_address)") _evolutions = { diff --git a/denyhosts_server/models.py b/denyhosts_server/models.py index 1159f29..d29642f 100644 --- a/denyhosts_server/models.py +++ b/denyhosts_server/models.py @@ -20,7 +20,7 @@ class ClientVersion(DBObject): - TABLENAME = 'ClientVersions' + TABLENAME = 'client_version' column_names = ['ip_address', 'first_time', 'latest_time', 'python_version', 'denyhosts_version', 'total_reports'] def __str__(self): From f010ace2d9b9c4f906114c0a43e19cd7bb386046 Mon Sep 17 00:00:00 2001 From: josev814 Date: Fri, 29 May 2020 12:39:48 -0400 Subject: [PATCH 4/5] updating to query client_version table in it's own method adding ClientVersion test --- denyhosts_server/controllers.py | 13 +++++++++---- tests/test_models.py | 23 ++++++++++++++++++++++- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/denyhosts_server/controllers.py b/denyhosts_server/controllers.py index 19fd240..8f361f9 100644 --- a/denyhosts_server/controllers.py +++ b/denyhosts_server/controllers.py @@ -50,6 +50,14 @@ def handle_report_from_client(client_ip, timestamp, hosts): utils.unlock_host(cracker_ip) logging.debug("Done adding report for {} from {}".format(cracker_ip,client_ip)) + +def get_client_version(client_ip): + return ClientVersion.find( + where=['ip_address=?', client_ip], + limit=1 + ) + + @inlineCallbacks def handle_version_report_from_client(client_ip, version_info, timestamp): if not utils.is_valid_ip_address(client_ip): @@ -59,10 +67,7 @@ def handle_version_report_from_client(client_ip, version_info, timestamp): dh_version = version_info[1] py_version = version_info[0] try: - client_report = yield ClientVersion.find( - where=['ip_address=?', client_ip], - limit=1 - ) + client_report = yield get_client_version(client_ip) if client_report is None: logging.debug("Adding version report for {}".format(client_ip)) save_version = ClientVersion( diff --git a/tests/test_models.py b/tests/test_models.py index d9bf655..87a478f 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -18,7 +18,7 @@ from denyhosts_server import models from denyhosts_server import controllers -from denyhosts_server.models import Cracker, Report +from denyhosts_server.models import Cracker, Report, ClientVersion from twisted.internet.defer import inlineCallbacks, returnValue @@ -119,4 +119,25 @@ def test_add_multiple_reports(self): cracker = yield controllers.get_cracker("192.168.1.1") self.assertIsNone(cracker, "Maintenance should remove cracker") + +class ClientVersionModelsTest(base.TestBase): + + client_ip = '64.233.185.100' + python_version = '2.7.14' + denyhosts_version = '3.1.2' + + def test_01_add_version_report(self): + now = time.time() + yield ClientVersion( + ip_address=self.client_ip, + first_time=now, + latest_time=now, + python_version=self.python_version, + denyhosts_version=self.denyhosts_version, + total_reports=1 + ).save() + + cv2 = yield controllers.get_client_version(self.client_ip) + yield self.assertIsNotNone(cv2, 'Added report is in database') + # vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4 From 5540965e191c523aaa2d89c6e6d87abb88fd9662 Mon Sep 17 00:00:00 2001 From: josev814 Date: Fri, 29 May 2020 12:51:45 -0400 Subject: [PATCH 5/5] updating peering to include send_client_version update method updating views to get the latest information from the db and passing that information to the peering servers --- denyhosts_server/peering.py | 27 ++++++++++++++++++++++++++- denyhosts_server/views.py | 3 ++- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/denyhosts_server/peering.py b/denyhosts_server/peering.py index a7314f4..f0dbb29 100644 --- a/denyhosts_server/peering.py +++ b/denyhosts_server/peering.py @@ -23,7 +23,7 @@ from xmlrpclib import ServerProxy from twisted.internet.defer import inlineCallbacks, returnValue -from twisted.internet.threads import deferToThread +from twisted.internet.threads import deferToThread import libnacl.public import libnacl.utils @@ -56,6 +56,30 @@ def send_update(client_ip, timestamp, hosts): except: logging.warning("Unable to send update to peer {}".format(peer)) +@inlineCallbacks +def send_client_version_update(client_info): + data = { + "ip_address": client_info.ip_address, + "first_time": client_info.first_time, + "latest_time": client_info.latest_time, + 'python_version': client_info.python_version, + 'denyhosts_version': client_info.denyhosts_version, + 'total_reports': client_info.total_reports + } + data_json = json.dumps(data) + + for peer in config.peers: + logging.debug("Sending client_version update to peer {}".format(peer)) + crypted = _peer_boxes[peer].encrypt(data_json) + base64 = crypted.encode('base64') + try: + server = yield deferToThread(ServerProxy, peer) + yield deferToThread(server.peering.update, _own_key.pk.encode('hex'), base64) + except Exception as e: + logging.exception('Error in Peer Version Reporting: {}'.format(peer)) + logging.debug("Done adding peer version reports from {}".format(client_ip)) + + def decrypt_message(peer_key, message): peer = None for _peer in config.peers: @@ -93,6 +117,7 @@ def handle_schema_version(peer_key, please): returnValue(schema_version) + @inlineCallbacks def handle_all_hosts(peer_key, please): data = decrypt_message(peer_key, please) diff --git a/denyhosts_server/views.py b/denyhosts_server/views.py index 46c9a8b..9995f5b 100644 --- a/denyhosts_server/views.py +++ b/denyhosts_server/views.py @@ -49,7 +49,8 @@ def xmlrpc_version_report(self, request, version_info): logging.info("version_report({}) from {}".format(version_info, remote_ip)) yield controllers.handle_version_report_from_client(remote_ip, version_info, now) try: - yield peering.send_version_update(remote_ip, version_info, now) + client_version_data = yield controllers.get_client_version(remote_ip) + yield peering.send_client_version_update(client_version_data) except xmlrpc.Fault, e: raise e except Exception, e: