From eda2e8298a85e0771ebab5631301e2650e6bfc77 Mon Sep 17 00:00:00 2001 From: peterjc Date: Wed, 26 Jun 2019 22:24:50 +0100 Subject: [PATCH 1/6] Define new command line API for key in environment variable The argument name --dkenv is short for deployment key environment variable, still considering better names for this. --- doctr/__main__.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/doctr/__main__.py b/doctr/__main__.py index 87b7782a..1b7db706 100644 --- a/doctr/__main__.py +++ b/doctr/__main__.py @@ -9,7 +9,8 @@ on your local machine. This will prompt for your GitHub credentials and the name of the repo you want to deploy docs for. This will generate a secure key, -which you should insert into your .travis.yml. +which you should insert into your .travis.yml (or set as a secure environment +variable in your TravisCI repository settings if using the --dkenv option). Then, on Travis, for the build where you build your docs, add:: @@ -134,6 +135,10 @@ def get_parser(config=None): if we do not appear to be on Travis.""") deploy_parser_add_argument('deploy_directory', type=str, nargs='?', help="""Directory to deploy the html documentation to on gh-pages.""") + deploy_parser_add_argument('--dkenv', type=str, metavar="ENVVAR", + help="""Push to GitHub using a deployment key stored in the named + environment variable via your TravisCI repository settings. + Use this if you used 'doctr configure --dkenv ENVVAR'.""") deploy_parser_add_argument('--token', action='store_true', default=False, help="""Push to GitHub using a personal access token. Use this if you used 'doctr configure --token'.""") @@ -193,6 +198,11 @@ def get_parser(config=None): configure_parser.set_defaults(func=configure) configure_parser.add_argument('--force', action='store_true', help="""Run the configure command even if we appear to be on Travis.""") + configure_parser.add_argument('--dkenv', type=str, metavar="ENVVAR", + help="""Generate a deployment key to push to GitHub. The public key + will be added to your GitHub repository settings. The private key + should be added to you TravisCI repository settings as a protected + environment variable.""") configure_parser.add_argument('--token', action="store_true", default=False, help="""Generate a personal access token to push to GitHub. The default is to use a deploy key. WARNING: This will grant read/write access to all the From 303d3462437bcd23a2dcbcc1d662802ea3887212 Mon Sep 17 00:00:00 2001 From: peterjc Date: Wed, 26 Jun 2019 23:13:26 +0100 Subject: [PATCH 2/6] First pass at the configure command Still need to fine tune how best to print the private key to screen (newlines etc). Next need to use the environment key in deploy. --- doctr/__main__.py | 65 +++++++++++++++++++++++++++++++++-------------- doctr/travis.py | 35 +++++++++++++++++++++---- 2 files changed, 76 insertions(+), 24 deletions(-) diff --git a/doctr/__main__.py b/doctr/__main__.py index 1b7db706..120413e2 100644 --- a/doctr/__main__.py +++ b/doctr/__main__.py @@ -285,6 +285,9 @@ def deploy(args, parser): config = get_config() + if args.token and args.dkenv: + parser.error("The --token and --dkenv settings are incompatible.") + if args.tmp_dir: parser.error("The --tmp-dir flag has been removed (doctr no longer uses a temporary directory when deploying).") @@ -320,10 +323,10 @@ def deploy(args, parser): canpush = setup_GitHub_push(deploy_repo, deploy_branch=deploy_branch, auth_type='token' if args.token else 'deploy_key', - full_key_path=keypath, + full_key_path=None if args.dkenv else keypath, branch_whitelist=branch_whitelist, build_tags=args.build_tags, - env_name=env_name) + env_name=args.dkenv if args.dkenv else env_name) if args.sync: built_docs = args.built_docs or find_sphinx_build_dir() @@ -393,6 +396,9 @@ def configure(args, parser): parser.error(red("doctr appears to be running on Travis. Use " "doctr configure --force to run anyway.")) + if args.token and args.dkenv: + parser.error("The --token and --dkenv settings are incompatible.") + if not args.authenticate: args.upload_key = False @@ -494,11 +500,17 @@ def configure(args, parser): deploy_key_repo, env_name, keypath = get_deploy_key_repo(deploy_repo, args.key_path) private_ssh_key, public_ssh_key = generate_ssh_key() - key = encrypt_to_file(private_ssh_key, keypath + '.enc') - del private_ssh_key # Prevent accidental use below + if args.dkenv: + key = None # don't need it on disk + encrypted_variable = None # not applicable + private_ssh_key = private_ssh_key.decode('ASCII') # Will print this later! + else: + key = encrypt_to_file(private_ssh_key, keypath + '.enc') + encrypted_variable = encrypt_variable(env_name.encode('utf-8') + b"=" + key, + build_repo=build_repo, tld=tld, + travis_token=travis_token, **login_kwargs) + private_ssh_key = None # Prevent accidental use below public_ssh_key = public_ssh_key.decode('ASCII') - encrypted_variable = encrypt_variable(env_name.encode('utf-8') + b"=" + key, - build_repo=build_repo, tld=tld, travis_token=travis_token, **login_kwargs) deploy_keys_url = 'https://github.com/{deploy_repo}/settings/keys'.format(deploy_repo=deploy_key_repo) @@ -523,12 +535,12 @@ def configure(args, parser): """.format(ssh_key=public_ssh_key, deploy_keys_url=deploy_keys_url, N=N, BOLD_MAGENTA=BOLD_MAGENTA, RESET=RESET))) + if not args.dkenv: + print(dedent("""\ + {N}. {BOLD_MAGENTA}Add the file {keypath}.enc to be staged for commit:{RESET} - print(dedent("""\ - {N}. {BOLD_MAGENTA}Add the file {keypath}.enc to be staged for commit:{RESET} - - git add {keypath}.enc - """.format(keypath=keypath, N=N, BOLD_MAGENTA=BOLD_MAGENTA, RESET=RESET))) + git add {keypath}.enc + """.format(keypath=keypath, N=N, BOLD_MAGENTA=BOLD_MAGENTA, RESET=RESET))) options = '--built-docs ' + bold_black('') if args.key_path: @@ -541,23 +553,38 @@ def configure(args, parser): options += ' --token' key_type = "personal access token" + if args.dkenv: + print(dedent("""\ + {N}. {BOLD_MAGENTA}Add the following private deployment key to your TravisCI + repository settings as environment variable {env_name}:{RESET} + + {private_ssh_key} + + WARNING: Do not otherwise share this private key! + """.format(N=N, BOLD_MAGENTA=BOLD_MAGENTA, RESET=RESET, + env_name=args.dkenv, private_ssh_key=private_ssh_key))) + print(dedent("""\ {N}. {BOLD_MAGENTA}Add these lines to your `.travis.yml` file:{RESET} + """.format(N=N, BOLD_MAGENTA=BOLD_MAGENTA, RESET=RESET))) - env: - global: - # Doctr {key_type} for {deploy_repo} - - secure: "{encrypted_variable}" + if not args.dkenv: + print(dedent("""\ + env: + global: + # Doctr {key_type} for {deploy_repo} + - secure: "{encrypted_variable}" + """.format(key_type=key_type, + encrypted_variable=encrypted_variable.decode('utf-8')))) + print(dedent("""\ script: - set -e - {BOLD_BLACK}{RESET} - pip install doctr - doctr deploy {options} {BOLD_BLACK}{RESET} - """.format(options=options, N=N, key_type=key_type, - encrypted_variable=encrypted_variable.decode('utf-8'), - deploy_repo=deploy_repo, BOLD_MAGENTA=BOLD_MAGENTA, - BOLD_BLACK=BOLD_BLACK, RESET=RESET))) + """.format(options=options, deploy_repo=deploy_repo, + BOLD_BLACK=BOLD_BLACK, RESET=RESET))) print(dedent("""\ Replace the text in {BOLD_BLACK}{RESET} with the relevant diff --git a/doctr/travis.py b/doctr/travis.py index 0f15eca5..7578a1a2 100644 --- a/doctr/travis.py +++ b/doctr/travis.py @@ -165,7 +165,10 @@ def get_current_repo(): 'remote.origin.url']).decode('utf-8') # Travis uses the https clone url - _, org, git_repo = remote_url.rsplit('.git', 1)[0].rsplit('/', 2) + try: + _, org, git_repo = remote_url.rsplit('.git', 1)[0].rsplit('/', 2) + except ValueError: + raise RuntimeError("Could not parse remote URL: %s" % remote_url) return (org + '/' + git_repo) def get_travis_branch(): @@ -189,12 +192,22 @@ def setup_GitHub_push(deploy_repo, *, auth_type='deploy_key', """ Setup the remote to push to GitHub (to be run on Travis). - ``auth_type`` should be either ``'deploy_key'`` or ``'token'``. + ``auth_type`` should be ``'deploy_key'`` (encrpted deplyoment key + on disk), ``'dkenv'`` (deployment key in environment variable), or + ``'token'``. - For ``auth_type='token'``, this sets up the remote with the token and - checks out the gh-pages branch. The token to push to GitHub is assumed to be in the ``GH_TOKEN`` environment + For ``auth_type='deploy_key'``, this sets up the remote with ssh access. + assuming the private deploement key is in the ``full_key_path`` file + and can be decrypted with the ``env_name`` environment variable. + + For ``auth_type='dkenv'``, this sets up the remote with ssh access + assuming the private deployment key is in the ``env_name`` environment variable. + For ``auth_type='token'``, this sets up the remote with the token and + checks out the gh-pages branch. The token to push to GitHub is assumed + to be in the ``GH_TOKEN`` environment variable. + For ``auth_type='deploy_key'``, this sets up the remote with ssh access. """ # Set to the name of the tag for tag builds @@ -220,7 +233,15 @@ def setup_GitHub_push(deploy_repo, *, auth_type='deploy_key', TRAVIS_REPO_SLUG = os.environ["TRAVIS_REPO_SLUG"] REPO_URL = 'https://api.github.com/repos/{slug}' r = requests.get(REPO_URL.format(slug=TRAVIS_REPO_SLUG)) - fork = r.json().get('fork', False) + + if auth_type == 'dkenv': + # Here we don't care if we are on a fork or not - one of the reasons + # for putting the key in a TravisCI secure environment variable is + # to allow things like setting up test source and deployment repos + # under a personal fork. + fork = False + else: + fork = r.json().get('fork', False) canpush = determine_push_rights( branch_whitelist=branch_whitelist, @@ -244,6 +265,10 @@ def setup_GitHub_push(deploy_repo, *, auth_type='deploy_key', run(['git', 'remote', 'add', 'doctr_remote', 'https://{token}@github.com/{deploy_repo}.git'.format(token=token.decode('utf-8'), deploy_repo=deploy_repo)]) + elif auth_type == 'dkenv': + # TODO - setup the key + run(['git', 'remote', 'add', 'doctr_remote', + 'git@github.com:{deploy_repo}.git'.format(deploy_repo=deploy_repo)]) else: keypath, key_ext = full_key_path.rsplit('.', 1) key_ext = '.' + key_ext From b8eb11ba8bf9d23a7736dee16f239ec69bf44c06 Mon Sep 17 00:00:00 2001 From: peterjc Date: Wed, 26 Jun 2019 23:34:00 +0100 Subject: [PATCH 3/6] Polishing config with --dkenv Getting the indentation to look nice (while deliberately not indenting the private key to make life simple for copy-and-paste). --- doctr/__main__.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/doctr/__main__.py b/doctr/__main__.py index 120413e2..bf38b131 100644 --- a/doctr/__main__.py +++ b/doctr/__main__.py @@ -399,6 +399,10 @@ def configure(args, parser): if args.token and args.dkenv: parser.error("The --token and --dkenv settings are incompatible.") + if len(args.dkenv.split()) != 1: + # Not going to repeat this sanity test in the deploy command: + parser.error("The --dkenv setting should be one word only, e.g. DOC_KEY.") + if not args.authenticate: args.upload_key = False @@ -531,6 +535,7 @@ def configure(args, parser): and add the following as a new key:{RESET} {ssh_key} + {BOLD_MAGENTA}Be sure to allow write access for the key.{RESET} """.format(ssh_key=public_ssh_key, deploy_keys_url=deploy_keys_url, N=N, BOLD_MAGENTA=BOLD_MAGENTA, RESET=RESET))) @@ -554,15 +559,13 @@ def configure(args, parser): key_type = "personal access token" if args.dkenv: + options += ' --dkenv ' + args.dkenv print(dedent("""\ {N}. {BOLD_MAGENTA}Add the following private deployment key to your TravisCI - repository settings as environment variable {env_name}:{RESET} - - {private_ssh_key} - - WARNING: Do not otherwise share this private key! + repository settings as environment variable {env_name}:{RESET} """.format(N=N, BOLD_MAGENTA=BOLD_MAGENTA, RESET=RESET, env_name=args.dkenv, private_ssh_key=private_ssh_key))) + print(private_ssh_key) print(dedent("""\ {N}. {BOLD_MAGENTA}Add these lines to your `.travis.yml` file:{RESET} From 5c9f2d8c6511060e315b41547494e2875d732ef9 Mon Sep 17 00:00:00 2001 From: peterjc Date: Thu, 27 Jun 2019 22:43:00 +0100 Subject: [PATCH 4/6] Cope with git or https clone url --- doctr/travis.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/doctr/travis.py b/doctr/travis.py index 7578a1a2..a9713602 100644 --- a/doctr/travis.py +++ b/doctr/travis.py @@ -165,11 +165,13 @@ def get_current_repo(): 'remote.origin.url']).decode('utf-8') # Travis uses the https clone url - try: - _, org, git_repo = remote_url.rsplit('.git', 1)[0].rsplit('/', 2) - except ValueError: - raise RuntimeError("Could not parse remote URL: %s" % remote_url) - return (org + '/' + git_repo) + # e.g. https://github.com//.git + # If run outside Travis using --force, might have: + # e.g. git@github.com:/.git + if remote_url.endswith(".git"): + remote_url = remote_url[:-4] + _, owner, repo = remote_url.replace(":", "/").rsplit("/", 2) + return owner + '/' + repo def get_travis_branch(): """Get the name of the branch that the PR is from. From 65e31de8c10e3cb9e3a04e46a7aefe8571423ab8 Mon Sep 17 00:00:00 2001 From: peterjc Date: Thu, 27 Jun 2019 22:48:04 +0100 Subject: [PATCH 5/6] Refactor decrypt_file to introduce write_private_key --- doctr/travis.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/doctr/travis.py b/doctr/travis.py index a9713602..32d7248c 100644 --- a/doctr/travis.py +++ b/doctr/travis.py @@ -20,6 +20,14 @@ from .common import red, blue, yellow DOCTR_WORKING_BRANCH = '__doctr_working_branch' +def write_private_key(filename, private_key_bytes): + """ + Helper function to record (decrpted) private key to as file for ssh. + """ + with open(filename, 'wb') as f: + f.write(filename) + os.chmod(filename, 0o600) + def decrypt_file(file, key): """ Decrypts the file ``file``. @@ -41,10 +49,7 @@ def decrypt_file(file, key): with open(file, 'rb') as f: decrypted_file = fer.decrypt(f.read()) - with open(file[:-4], 'wb') as f: - f.write(decrypted_file) - - os.chmod(file[:-4], 0o600) + write_private_key(file[:-4], decrypted_file) def setup_deploy_key(keypath='github_deploy_key', key_ext='.enc', env_name='DOCTR_DEPLOY_ENCRYPTION_KEY'): """ From 6b304709f8fc8e54c01a183113d556de8772a08c Mon Sep 17 00:00:00 2001 From: peterjc Date: Thu, 27 Jun 2019 23:22:17 +0100 Subject: [PATCH 6/6] Write the --dkenv private key to disk --- doctr/travis.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/doctr/travis.py b/doctr/travis.py index 32d7248c..76389650 100644 --- a/doctr/travis.py +++ b/doctr/travis.py @@ -28,6 +28,12 @@ def write_private_key(filename, private_key_bytes): f.write(filename) os.chmod(filename, 0o600) +def write_key_from_env_var(filename, env_var): + """ + Write private key from environment variable named in --dkenv to disk. + """ + write_private_key(filename, os.environ[env_var]) + def decrypt_file(file, key): """ Decrypts the file ``file``. @@ -230,8 +236,8 @@ def setup_GitHub_push(deploy_repo, *, auth_type='deploy_key', stacklevel=2) branch_whitelist.add('master') - if auth_type not in ['deploy_key', 'token']: - raise ValueError("auth_type must be 'deploy_key' or 'token'") + if auth_type not in ['deploy_key', 'dkenv', 'token']: + raise ValueError("auth_type must be 'deploy_key', 'dkenv', or 'token'") TRAVIS_BRANCH = os.environ.get("TRAVIS_BRANCH", "") TRAVIS_PULL_REQUEST = os.environ.get("TRAVIS_PULL_REQUEST", "") @@ -258,6 +264,14 @@ def setup_GitHub_push(deploy_repo, *, auth_type='deploy_key', TRAVIS_TAG=TRAVIS_TAG, build_tags=build_tags) + if auth_type == 'dkenv': + if args.dkenv not in os.environ: + print("WARNING: Environment variable {dkenv} not set".format(dpenv=args.dkenv)) + canpush = False + elif not os.environ[args.dkenv]: + print("WARNING: Environment variable {dkenv} empty".format(dpenv=args.dkenv)) + canpush = False + print("Setting git attributes") set_git_user_email() @@ -274,6 +288,7 @@ def setup_GitHub_push(deploy_repo, *, auth_type='deploy_key', deploy_repo=deploy_repo)]) elif auth_type == 'dkenv': # TODO - setup the key + write_key_from_env_var(full_key_path.rsplit('.', 1)[0], args.dkenv) run(['git', 'remote', 'add', 'doctr_remote', 'git@github.com:{deploy_repo}.git'.format(deploy_repo=deploy_repo)]) else: