From e61fa1957a813ff2861fd2034848a0f6db12fdbb Mon Sep 17 00:00:00 2001 From: tim Date: Tue, 4 May 2010 11:43:50 +0800 Subject: [PATCH 01/11] add user --- mr/awsome/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/mr/awsome/__init__.py b/mr/awsome/__init__.py index c8d431d..df2712b 100644 --- a/mr/awsome/__init__.py +++ b/mr/awsome/__init__.py @@ -253,8 +253,13 @@ def init_ssh_key(self, user=None): if instance is None: log.error("Can't establish ssh connection.") return + if user is None: - user = 'root' + # check user at config file + user = self.config['user'] + if user is None: + user = 'root' + host = str(instance.public_dns_name) port = 22 client = paramiko.SSHClient() @@ -690,4 +695,4 @@ def aws_ssh(configpath=None): argv = sys.argv[:] argv.insert(1, "ssh") aws = AWS(configfile=configpath) - return aws(argv) \ No newline at end of file + return aws(argv) From a93a9d60e75e51cc959c02ca52d42331ea7ca7f6 Mon Sep 17 00:00:00 2001 From: cocoy Date: Wed, 19 May 2010 13:20:47 +0800 Subject: [PATCH 02/11] Revert "add user" This reverts commit e61fa1957a813ff2861fd2034848a0f6db12fdbb. --- mr/awsome/__init__.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/mr/awsome/__init__.py b/mr/awsome/__init__.py index df2712b..c8d431d 100644 --- a/mr/awsome/__init__.py +++ b/mr/awsome/__init__.py @@ -253,13 +253,8 @@ def init_ssh_key(self, user=None): if instance is None: log.error("Can't establish ssh connection.") return - if user is None: - # check user at config file - user = self.config['user'] - if user is None: - user = 'root' - + user = 'root' host = str(instance.public_dns_name) port = 22 client = paramiko.SSHClient() @@ -695,4 +690,4 @@ def aws_ssh(configpath=None): argv = sys.argv[:] argv.insert(1, "ssh") aws = AWS(configfile=configpath) - return aws(argv) + return aws(argv) \ No newline at end of file From bb421957494cfaa3a9af187b1692b689199a6358 Mon Sep 17 00:00:00 2001 From: cocoy Date: Wed, 19 May 2010 13:46:35 +0800 Subject: [PATCH 03/11] change back to server_user and added some items on README --- README.txt | 45 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/README.txt b/README.txt index be48ce8..c27ba14 100644 --- a/README.txt +++ b/README.txt @@ -9,11 +9,22 @@ Examples are adding additional, pre-configured webservers to a cluster deployments and creating backups - each with just one call from the commandline. Aw(e)some, indeed, if we may say so... +** Requirements ** + Python2.6 + **Installation** mr.awsome is best installed with easy_install, pip or with zc.recipe.egg in a buildout. It installs two scripts, ``aws`` and ``assh``. +A simple directory structure for a project: + ec2-project/ + etc/ + aws.conf + fabfile.py + dbstartup.sh + webstartup.sh + **Configuration** To authorize itself against AWS, mr.awsome uses the following two environment @@ -28,6 +39,10 @@ You can also put them into files and point to them in the ``[aws]`` section with the ``access-key-id`` and ``secret-access-key`` options. It's best to put them in ``~/.aws/`` and make sure only your user can read them. + [aws] + access-key-id = /home/user/.aws/access-file + secret-access-key = /home/user/.aws/secret-file + All other information about server instances is located in ``aws.conf``, which by default is looked up in ``etc/aws.conf``. @@ -129,7 +144,35 @@ Directly after that follows the binary data of the gzipped startup script. **Snapshots** -(Needs description of volumes in "Configuration") +** EBS Volumes ** +To attach EBS volumes : + + [instance:demo-server] + keypair = default + securitygroups = demo-server + region = eu-west-1 + placement = eu-west-1a + # we use images from `http://alestic.com/`_ + image = ami-a62a01d2 + startup_script = startup-demo-server + fabfile = fabfile.py + volumes = + vol-xxxxx /dev/sdh + vol-yyyyy /dev/sdg + +** Elastic IP *** +You have to allocate the new IP and use it to the instance. The tool will associate the Elastic IP to the instance. + + [instance:demo-server] + keypair = default + securitygroups = demo-server + region = eu-west-1 + placement = eu-west-1a + # we use images from `http://alestic.com/`_ + # Ubuntu 9.10 Karmic server 32-bit Europe + image = ami-a62a01d2 + ip = xxx.xxx.xxx.xxx + **SSH integration** From cfcce3865437ff0e1ad6a1d0ea990755aa5f3884 Mon Sep 17 00:00:00 2001 From: cocoy Date: Wed, 19 May 2010 13:46:51 +0800 Subject: [PATCH 04/11] change back to server_user and added some items on README --- mr/awsome/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mr/awsome/__init__.py b/mr/awsome/__init__.py index c8d431d..345f2b4 100644 --- a/mr/awsome/__init__.py +++ b/mr/awsome/__init__.py @@ -254,7 +254,10 @@ def init_ssh_key(self, user=None): log.error("Can't establish ssh connection.") return if user is None: - user = 'root' + # check user at config file + user = self.config['server_user'] + if user is None: + user = 'root' host = str(instance.public_dns_name) port = 22 client = paramiko.SSHClient() @@ -690,4 +693,4 @@ def aws_ssh(configpath=None): argv = sys.argv[:] argv.insert(1, "ssh") aws = AWS(configfile=configpath) - return aws(argv) \ No newline at end of file + return aws(argv) From 5fb2d1cadd6fe40aed9bffc6a8c1c141c6ef9f45 Mon Sep 17 00:00:00 2001 From: cocoy Date: Fri, 21 May 2010 11:55:14 +0800 Subject: [PATCH 05/11] added reboot option --- mr/awsome/__init__.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/mr/awsome/__init__.py b/mr/awsome/__init__.py index 345f2b4..696e6ab 100644 --- a/mr/awsome/__init__.py +++ b/mr/awsome/__init__.py @@ -426,6 +426,29 @@ def cmd_stop(self, argv, help): return log.info("Instance stopped") + def cmd_reboot(self, argv, help): + """Reboot the instance""" + parser = argparse.ArgumentParser( + prog="aws reboot", + description=help, + ) + parser.add_argument("server", nargs=1, + metavar="instance", + help="Name of the instance from the config.", + choices=list(self.ec2.instances)) + args = parser.parse_args(argv) + server = self.ec2.instances[args.server[0]] + instance = server.instance + if instance is None: + return + if instance.state != 'running': + log.info("Instance state: %s", instance.state) + log.info("Instance not terminated") + return + rc = server.conn.reboot_instances([instance.id]) + instance._update(rc[0]) + log.info("Instance rebooting") + def cmd_terminate(self, argv, help): """Terminates the instance""" parser = argparse.ArgumentParser( @@ -641,7 +664,7 @@ def cmd_ssh(self, argv, help): def cmd_snapshot(self, argv, help): """Creates a snapshot of the volumes specified in the configuration""" parser = argparse.ArgumentParser( - prog="aws status", + prog="aws snapshot", description=help, ) parser.add_argument("server", nargs=1, From 938022197033b150e415b3d029b5103e7506609c Mon Sep 17 00:00:00 2001 From: cocoy Date: Fri, 21 May 2010 12:47:52 +0800 Subject: [PATCH 06/11] remove instance._update --- mr/awsome/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mr/awsome/__init__.py b/mr/awsome/__init__.py index 696e6ab..4f12fd5 100644 --- a/mr/awsome/__init__.py +++ b/mr/awsome/__init__.py @@ -446,7 +446,7 @@ def cmd_reboot(self, argv, help): log.info("Instance not terminated") return rc = server.conn.reboot_instances([instance.id]) - instance._update(rc[0]) + #instance._update(rc[0]) log.info("Instance rebooting") def cmd_terminate(self, argv, help): From 50b64126e90e726e5b61864c5530e292c764fd31 Mon Sep 17 00:00:00 2001 From: cocoy Date: Fri, 21 May 2010 16:42:36 +0800 Subject: [PATCH 07/11] fix error for undefined hosts for aws do and aws ssh --- mr/awsome/__init__.py | 43 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/mr/awsome/__init__.py b/mr/awsome/__init__.py index 4f12fd5..e0225a1 100644 --- a/mr/awsome/__init__.py +++ b/mr/awsome/__init__.py @@ -564,6 +564,23 @@ def cmd_do(self, argv, help): old_sys_argv = sys.argv old_cwd = os.getcwd() + # check server if active..else return . + tmpserver = None + try: + tmpserver = self.ec2.instances[argv[0]] + except KeyError,e: + log.error("Server not found %s", argv[0]) + parser.parse_args(argv[0]) + return + + if tmpserver is not None: + instance = tmpserver.instance + if instance is None: + return + if instance.state != 'running': + log.info("Instance state: %s", instance.state) + return + import fabric_integration # this needs to be done before any other fabric module import fabric_integration.patch() @@ -572,11 +589,14 @@ def cmd_do(self, argv, help): import fabric.main hoststr = None + try: fabric_integration.ec2 = self.ec2 fabric_integration.log = log hoststr = argv[0] server = self.ec2.all[hoststr] + + # prepare the connection fabric.state.env.reject_unknown_hosts = True fabric.state.env.disable_known_hosts = True @@ -646,14 +666,23 @@ def cmd_ssh(self, argv, help): if sid_index is None: parser.print_help() return - server = self.ec2.all[argv[sid_index]] - try: - user, host, port, client, known_hosts = server.init_ssh_key() + + hoststr = argv[sid_index] + server = None + try: + server = self.ec2.all[argv[sid_index]] + except KeyError,e: + parser.parse_args([hoststr]) + return + + try: + user, host, port, client, known_hosts = server.init_ssh_key() except paramiko.SSHException, e: - log.error("Couldn't validate fingerprint for ssh connection.") - log.error(e) - log.error("Is the server finished starting up?") - return + log.error("Couldn't validate fingerprint for ssh connection.") + log.error(e) + log.error("Is the server finished starting up?") + return + client.close() argv[sid_index:sid_index+1] = ['-o', 'UserKnownHostsFile=%s' % known_hosts, '-l', user, From 8633250b46b1d017fedc52c4e64d3cd68fa5a99a Mon Sep 17 00:00:00 2001 From: cocoy Date: Fri, 21 May 2010 18:05:17 +0800 Subject: [PATCH 08/11] Fix aws ssh host undefined. --- mr/awsome/__init__.py | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/mr/awsome/__init__.py b/mr/awsome/__init__.py index e0225a1..e60b4b5 100644 --- a/mr/awsome/__init__.py +++ b/mr/awsome/__init__.py @@ -570,7 +570,7 @@ def cmd_do(self, argv, help): tmpserver = self.ec2.instances[argv[0]] except KeyError,e: log.error("Server not found %s", argv[0]) - parser.parse_args(argv[0]) + parser.parse_args([argv[0]]) return if tmpserver is not None: @@ -675,8 +675,36 @@ def cmd_ssh(self, argv, help): parser.parse_args([hoststr]) return + ## end check running server + tmpserver = None + tmpserver = self.ec2.instances[argv[0]] + + if tmpserver is not None: + instance = tmpserver.instance + if instance is None: + return + if instance.state != 'running': + log.info("Instance state: %s", instance.state) + return + ## end check running server + try: - user, host, port, client, known_hosts = server.init_ssh_key() + if server is not None: + ## end check running server + tmpserver = None + tmpserver = self.ec2.instances[argv[0]] + + if tmpserver is not None: + instance = tmpserver.instance + if instance is None: + return + if instance.state != 'running': + log.info("Instance state: %s", instance.state) + return + ## end check running server + user, host, port, client, known_hosts = server.init_ssh_key() + else: + return except paramiko.SSHException, e: log.error("Couldn't validate fingerprint for ssh connection.") log.error(e) From 8083d7c0a2fe0edce93622a950b62ba975a7f523 Mon Sep 17 00:00:00 2001 From: cocoy Date: Tue, 25 May 2010 00:30:52 +0800 Subject: [PATCH 09/11] Add cluster commands: cluster_list,cluster_start,cluster_terminate. --- mr/awsome/__init__.py | 84 +++++++++++++++++++++++++++++++++++++++++++ mr/awsome/config.py | 9 +++++ 2 files changed, 93 insertions(+) diff --git a/mr/awsome/__init__.py b/mr/awsome/__init__.py index e60b4b5..fca05db 100644 --- a/mr/awsome/__init__.py +++ b/mr/awsome/__init__.py @@ -291,6 +291,13 @@ def snapshot(self, devs=None): volume.create_snapshot(description=description) +class Cluster(object): + def __init__(self,ec2, sid): + self.id = sid + self.ec2 = ec2 + self.config = self.ec2.config['cluster'][sid] + + class Server(object): def __init__(self, ec2, sid): self.id = sid @@ -344,6 +351,9 @@ def __init__(self, configpath): self.all.update(self.instances) self.all.update(self.servers) + self.clusters = {} + for cid in self.config.get('cluster',{}): + self.clusters[cid] = Cluster(self,cid) class AWS(object): def __init__(self, configfile=None): @@ -488,6 +498,80 @@ def _parse_overrides(self, options): overrides[key] = value return overrides + def cmd_cluster_list (self, argv, help): + """List servers on a cluster""" + parser = argparse.ArgumentParser( + prog="aws cluster_list", + description=help, + ) + parser.add_argument("cluster", nargs=1, + metavar="cluster", + help="Name of the cluster from the config.", + choices=list(self.ec2.clusters)) + + args = parser.parse_args(argv) + # get cluster + cluster = self.ec2.clusters[args.cluster[0]] + servers = cluster.config.get('servers',[]) + log.info("---Listing servers at cluster %s.",cluster.id) + i = 1 + for s in servers: + log.info("---Cluster %s server %i: %s.---",cluster.id,i,s) + server = self.ec2.instances[s] + self._status(server) + i = i + 1 + + def cmd_cluster_start (self, argv, help): + """Starts the cluster of servers""" + parser = argparse.ArgumentParser( + prog="aws cluster_start", + description=help, + ) + parser.add_argument("cluster", nargs=1, + metavar="cluster", + help="Name of the cluster from the config.", + choices=list(self.ec2.clusters)) + + args = parser.parse_args(argv) + # get cluster + cluster = self.ec2.clusters[args.cluster[0]] + servers = cluster.config.get('servers',[]) + log.info("---Start server at cluster %s",cluster.id) + for s in servers: + log.info("--%s:%s", cluster.id, s) + server = self.ec2.instances[s] + opts = server.config.copy() + instance = server.start(opts) + self._status(server) + + def cmd_cluster_terminate (self, argv, help): + """Terminate the cluster of servers""" + parser = argparse.ArgumentParser( + prog="aws cluster_terminate", + description=help, + ) + parser.add_argument("cluster", nargs=1, + metavar="cluster", + help="Name of the cluster from the config.", + choices=list(self.ec2.clusters)) + + args = parser.parse_args(argv) + cluster = self.ec2.clusters[args.cluster[0]] + servers = cluster.config.get('servers',[]) + log.info("---Terminating servers at cluster %s.",cluster.id) + for s in servers: + server = self.ec2.instances[s] + instance = server.instance + if instance is None: + continue + if instance.state != 'running': + log.info("Instance state: %s", instance.state) + log.info("Instance not terminated") + continue + rc = server.conn.terminate_instances([instance.id]) + instance._update(rc[0]) + log.info("Instance terminated") + def cmd_start(self, argv, help): """Starts the instance""" parser = argparse.ArgumentParser( diff --git a/mr/awsome/config.py b/mr/awsome/config.py index 05bef03..084d2cd 100644 --- a/mr/awsome/config.py +++ b/mr/awsome/config.py @@ -33,6 +33,15 @@ def massage_instance_volumes(self, value): volumes.append((volume[0], volume[1])) return tuple(volumes) + def massage_cluster_servers(self, value): + servers = [] + for line in value.split('\n'): + server = line.split() + if not len(server): + continue + servers.append((server[0])) + return tuple(servers) + def massage_securitygroup_connections(self, value): connections = [] for line in value.split('\n'): From 9c248558a4e42872b5df4f19c8c36288dbe32c8d Mon Sep 17 00:00:00 2001 From: cocoy Date: Wed, 26 May 2010 21:07:15 +0800 Subject: [PATCH 10/11] Update README add Cluster option. --- README.txt | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/README.txt b/README.txt index c27ba14..3c12353 100644 --- a/README.txt +++ b/README.txt @@ -174,6 +174,23 @@ You have to allocate the new IP and use it to the instance. The tool will associ ip = xxx.xxx.xxx.xxx +** Cluster *** + By defining a cluster in etc/aws.conf: + + [cluster:lamp-cluster] + servers = + demo-webserver + demo-dbserver + + This will be capable of starting the two servers in one command: + + $ aws cluster_start lamp-cluster + + And terminating them also in one go: + + $ aws cluster_terminate lamp-cluster + + **SSH integration** mr.awsome provides an additional tool ``assh`` to easily perform SSH based From 9434712c50038f407abbff6051e53248a3ed329d Mon Sep 17 00:00:00 2001 From: cocoy Date: Mon, 31 May 2010 13:58:13 +0800 Subject: [PATCH 11/11] Minor fixed on macro,security group. --- mr/awsome/config.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/mr/awsome/config.py b/mr/awsome/config.py index 084d2cd..abba079 100644 --- a/mr/awsome/config.py +++ b/mr/awsome/config.py @@ -10,12 +10,16 @@ def massage_instance_fabfile(self, value): def massage_instance_startup_script(self, value): result = dict() - if value.startswith('gzip:'): - value = value[5:] - result['gzip'] = True - if not os.path.isabs(value): - value = os.path.join(self.path, value) - result['path'] = value + ## value is already a dict, from macro + if isinstance(value,dict) is True: + return value + else: + if value.startswith('gzip:'): + value = value[5:] + result['gzip'] = True + if not os.path.isabs(value): + value = os.path.join(self.path, value) + result['path'] = value return result def massage_instance_securitygroups(self, value): @@ -65,6 +69,7 @@ def _expand(self, sectiongroupname, sectionname, section, seen): raise ValueError("Circular macro expansion.") macrogroupname = sectiongroupname macroname = section['<'] + seen.add((sectiongroupname, sectionname)) if ':' in macroname: macrogroupname, macroname = macroname.split(':')