From d008b8b9881c2911b117769308468f9786da901b Mon Sep 17 00:00:00 2001 From: Martijn Jacobs Date: Tue, 15 Dec 2020 22:33:29 +0100 Subject: [PATCH 01/22] Fix test output with newer version of click --- vdt/simpleaptrepo/tests/test_cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vdt/simpleaptrepo/tests/test_cli.py b/vdt/simpleaptrepo/tests/test_cli.py index a12aa64..e95dffb 100644 --- a/vdt/simpleaptrepo/tests/test_cli.py +++ b/vdt/simpleaptrepo/tests/test_cli.py @@ -29,8 +29,9 @@ """ CREATE_REPO_NO_PARAMS_OUTPUT = """Usage: create-repo [OPTIONS] NAME [PATH] +Try 'create-repo --help' for help. -Error: Missing argument "name". +Error: Missing argument 'NAME'. """ LIST_REPOS_OUTPUT = """my_repo (gpgkey: 123456) From a89eb7797b38a906dd9ac46e883d044ecd9a8cfa Mon Sep 17 00:00:00 2001 From: Martijn Jacobs Date: Tue, 15 Dec 2020 22:35:26 +0100 Subject: [PATCH 02/22] Updated python version (3.6 / 3.7 / 3.8) --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 292758c..8faf0c2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,12 @@ language: python python: - - "2.7" - - "3.5" - "3.6" + - "3.7" + - "3.8" services: - docker - + install: - make install From 8c31913fb9034f79cd8fcfc3c8307c466ffff1b5 Mon Sep 17 00:00:00 2001 From: Martijn Jacobs Date: Tue, 15 Dec 2020 22:36:05 +0100 Subject: [PATCH 03/22] Use debian 9 for the test image, use python3-pip And remove the entrypoint --- docker/Dockerfile | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 27bcd27..b08d513 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,12 +1,10 @@ -FROM ubuntu -MAINTAINER Martijn Jacobs +FROM debian:9 +MAINTAINER Martijn Jacobs ENV DEBIAN_FRONTEND noninteractive # install the debian related packages for this tool + python pip -RUN apt-get update && apt-get install python-pip gnupg dpkg-sig apt-utils --yes +RUN apt-get update && apt-get install python3-pip gnupg dpkg-sig apt-utils --yes # Make sure you mounted the source coude in /usr/local/src -WORKDIR /usr/local/src/ - -ENTRYPOINT ["make", "test"] \ No newline at end of file +WORKDIR /usr/local/src/ \ No newline at end of file From 20975335b12c261ce2f172c382c04d99ec223178 Mon Sep 17 00:00:00 2001 From: Martijn Jacobs Date: Tue, 15 Dec 2020 22:36:27 +0100 Subject: [PATCH 04/22] Use new GPG key, use python3 here --- docker/entrypoint.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 6478168..f13e462 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -1,16 +1,22 @@ #!/bin/bash +export LC_ALL=C.UTF-8 +export LANG=C.UTF-8 + # runs the unittests -pip install pip --upgrade -make test +pip3 install pip --upgrade +pip3 install -e .[dev] +make nosetest + # do some cleanup if [ -d "/repos/" ]; then - rm -Rf /repos/ + rm -Rf /repos/ fi mkdir /repos + # # now we really create a repo here and add a package # # the gpg key is build into the docker image for testing purposes -simpleapt create-repo myrepo /repos --gpgkey D2E0953C +simpleapt create-repo myrepo /repos --gpgkey AC46C6AAD5792E7E simpleapt add-component myrepo test simpleapt list-repos cp /usr/local/src/vdt/simpleaptrepo/tests/testdata/*.deb /repos/myrepo/test From 745760ac02866c5c739bd9727f121f20d6ca7c26 Mon Sep 17 00:00:00 2001 From: Martijn Jacobs Date: Tue, 15 Dec 2020 22:36:43 +0100 Subject: [PATCH 05/22] Remove unneeded TODO --- TODO.txt | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 TODO.txt diff --git a/TODO.txt b/TODO.txt deleted file mode 100644 index 4829c54..0000000 --- a/TODO.txt +++ /dev/null @@ -1,4 +0,0 @@ -- test with python 3.5 -- add unittests -- add coverage.io -- add to travis From b09da632b7a545616f04932e580b869773fe1c70 Mon Sep 17 00:00:00 2001 From: Martijn Jacobs Date: Tue, 15 Dec 2020 22:36:55 +0100 Subject: [PATCH 06/22] Updated setup.py with new info --- setup.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 6db9f2a..5955b60 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -__version__ = "0.0.5" +__version__ = "1.0.0" setup( @@ -14,17 +14,15 @@ 'License :: OSI Approved :: BSD License', 'Operating System :: Unix', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', - + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], keywords='', author='Martijn Jacobs', - author_email='martijn@devopsconsulting.nl', + author_email='maerteijn@gmail.com', url='https://github.com/devopsconsulting/vdt.simpleaptrepo', license='BSD', # include all packages in the egg, except the test package. @@ -49,4 +47,5 @@ extras_require={ 'dev': ['nose', 'coverage', 'mock', 'twine'], }, + python_requires='>=3.6', ) From dec47e1f25c40f79cb5822ccddde7ef569a9e36f Mon Sep 17 00:00:00 2001 From: Martijn Jacobs Date: Tue, 15 Dec 2020 22:37:02 +0100 Subject: [PATCH 07/22] Drop python 2 support --- vdt/simpleaptrepo/config.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/vdt/simpleaptrepo/config.py b/vdt/simpleaptrepo/config.py index 78e6b50..511dbcc 100644 --- a/vdt/simpleaptrepo/config.py +++ b/vdt/simpleaptrepo/config.py @@ -1,7 +1,4 @@ -try: - import ConfigParser # python 2 -except: - import configparser as ConfigParser # python 3 +import configparser as ConfigParser import os From 09014ed069a22b84ff2eac69acbbaed68519d541 Mon Sep 17 00:00:00 2001 From: Martijn Jacobs Date: Tue, 15 Dec 2020 22:37:22 +0100 Subject: [PATCH 08/22] Use SHA256 as key (required in debian 9) --- vdt/simpleaptrepo/repo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vdt/simpleaptrepo/repo.py b/vdt/simpleaptrepo/repo.py index 6f29065..6107e61 100644 --- a/vdt/simpleaptrepo/repo.py +++ b/vdt/simpleaptrepo/repo.py @@ -17,7 +17,7 @@ def create_gpg_key(output_command): def export_pubkey(path, gpgkey, output_command): key_path = os.path.join(path, 'keyfile') - cmd = "/usr/bin/gpg --yes --output %s --armor --export %s" % ( + cmd = "/usr/bin/gpg --digest-algo SHA256 --yes --output %s --armor --export %s" % ( key_path, gpgkey) subprocess.check_output(cmd, shell=True) output_command("Exported key %s to %s" % (gpgkey, key_path)) @@ -74,7 +74,7 @@ def create_signed_releases_index(path, gpgkey, output_command): "/usr/bin/apt-ftparchive release . > Release", shell=True, cwd=path) output_command("Create InRelease with key %s" % gpgkey) subprocess.check_output( - "/usr/bin/gpg --yes -u 0x%s --clearsign -o InRelease Release" % ( + "/usr/bin/gpg --digest-algo SHA256 --yes -u 0x%s --clearsign -o InRelease Release" % ( gpgkey), shell=True, cwd=path) output_command("Create Releases.gpg with key %s" % gpgkey) subprocess.check_output( From c1a4389c0a4ed85502680940e73f8328763a7685 Mon Sep 17 00:00:00 2001 From: Martijn Jacobs Date: Tue, 15 Dec 2020 22:39:02 +0100 Subject: [PATCH 09/22] Make this python3 compatible --- vdt/simpleaptrepo/repo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vdt/simpleaptrepo/repo.py b/vdt/simpleaptrepo/repo.py index 6107e61..571ed7e 100644 --- a/vdt/simpleaptrepo/repo.py +++ b/vdt/simpleaptrepo/repo.py @@ -34,7 +34,7 @@ def sign_packages(path, gpgkey, skip_signed, output_command): # but still could have run succesful output = e.output - if "_gpgbuilder" in output: + if b"_gpgbuilder" in output: if skip_signed: output_command("Skipped signing %s" % deb_file) From 3fc995a7670a4717c48d88197d8c0c4638fa4ef8 Mon Sep 17 00:00:00 2001 From: Martijn Jacobs Date: Tue, 15 Dec 2020 23:01:29 +0100 Subject: [PATCH 10/22] Update build.sh to use --digest-algo SHA256 --- docker/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/build.sh b/docker/build.sh index ea6e0a8..2029320 100644 --- a/docker/build.sh +++ b/docker/build.sh @@ -4,7 +4,7 @@ docker build -t maerteijn/simpeapt-test-image . # run a interactive shell and you should generate a gpg key -echo "Please run /usr/bin/gpg --gen-key, then exit the container with ctrl+D" +echo "Please run /usr/bin/gpg --digest-algo SHA256 --gen-key, then exit the container with ctrl+D" docker run -it --name=simpleapt-test-image-generate-key maerteijn/simpeapt-test-image /bin/bash echo "Please remember the generated GPG key hash" From c64eb0c2c3de3034a1610c305ac70d91006b80a8 Mon Sep 17 00:00:00 2001 From: Martijn Jacobs Date: Wed, 16 Dec 2020 10:40:45 +0100 Subject: [PATCH 11/22] Restructure and cleanup the invokation of the docker image and tests --- .travis.yml | 3 +-- Makefile | 8 +++++--- docker/Dockerfile | 2 +- docker/{build.sh => build-test-image.sh} | 2 +- docker/docker-compose.yml | 7 ------- docker/{entrypoint.sh => run-tests.sh} | 4 ++-- 6 files changed, 10 insertions(+), 16 deletions(-) rename docker/{build.sh => build-test-image.sh} (86%) mode change 100644 => 100755 delete mode 100644 docker/docker-compose.yml rename docker/{entrypoint.sh => run-tests.sh} (80%) diff --git a/.travis.yml b/.travis.yml index 8faf0c2..72bf0f4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,5 +11,4 @@ install: - make install script: - - make test - - docker-compose -f docker/docker-compose.yml up + - make docker-test diff --git a/Makefile b/Makefile index 41e77cb..40dff9b 100644 --- a/Makefile +++ b/Makefile @@ -1,15 +1,17 @@ clean: - find . -name '*.pyc' -delete find . -name '__pycache__' -delete find . -name '*.egg-info' -delete - + nosetest: nosetests -s --with-coverage --cover-erase --cover-package=vdt.simpleaptrepo --cover-xml --logging-level=INFO --with-doctest --verbosity=2 install: pip install -e .[dev] -test: install nosetest +test: nosetest + +docker-test: + docker run --rm -v $(PWD):/usr/local/src maerteijn/simpeapt-test-image ./docker/run-tests.sh clean_release: clean if [ -d "dist" ]; then rm dist/*; fi diff --git a/docker/Dockerfile b/docker/Dockerfile index b08d513..9f0018e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:9 +FROM debian:10 MAINTAINER Martijn Jacobs ENV DEBIAN_FRONTEND noninteractive diff --git a/docker/build.sh b/docker/build-test-image.sh old mode 100644 new mode 100755 similarity index 86% rename from docker/build.sh rename to docker/build-test-image.sh index 2029320..d7438df --- a/docker/build.sh +++ b/docker/build-test-image.sh @@ -13,4 +13,4 @@ docker commit simpleapt-test-image-generate-key maerteijn/simpeapt-test-image echo "Done!" echo "Now push the image with 'docker push maerteijn/simpeapt-test-image'" -echo "and delete the container afterwards: 'docker rm simpleapt-test-image-generate-key'" +echo "and delete the container afterwards: 'docker rm simpleapt-test-image-generate-key'" diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml deleted file mode 100644 index 7d35ad3..0000000 --- a/docker/docker-compose.yml +++ /dev/null @@ -1,7 +0,0 @@ -version: '3' -services: - simpaltrepo-test: - image: maerteijn/simpeapt-test-image - volumes: - - ..:/usr/local/src - entrypoint: ./docker/entrypoint.sh diff --git a/docker/entrypoint.sh b/docker/run-tests.sh similarity index 80% rename from docker/entrypoint.sh rename to docker/run-tests.sh index f13e462..0285dd3 100755 --- a/docker/entrypoint.sh +++ b/docker/run-tests.sh @@ -14,8 +14,8 @@ fi mkdir /repos -# # now we really create a repo here and add a package -# # the gpg key is build into the docker image for testing purposes +# now we really create a repo here and add a package +# the gpg key is build into the docker image for testing purposes simpleapt create-repo myrepo /repos --gpgkey AC46C6AAD5792E7E simpleapt add-component myrepo test simpleapt list-repos From 80d6c8890b98b6695db4c85df6b162cc1a7da6e5 Mon Sep 17 00:00:00 2001 From: Martijn Jacobs Date: Wed, 16 Dec 2020 10:42:45 +0100 Subject: [PATCH 12/22] Changed license to MIT and added a license file --- LICENSE | 21 +++++++++++++++++++++ setup.py | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..369a437 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2016-2021 Martijn Jacobs (maerteijn) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/setup.py b/setup.py index 5955b60..5340f8c 100755 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ author='Martijn Jacobs', author_email='maerteijn@gmail.com', url='https://github.com/devopsconsulting/vdt.simpleaptrepo', - license='BSD', + license='MIT', # include all packages in the egg, except the test package. packages=find_packages( exclude=['ez_setup', 'examples', '*tests']), From 1c761d357d655af97bd4dc12c47b29512c954bd4 Mon Sep 17 00:00:00 2001 From: Martijn Jacobs Date: Wed, 16 Dec 2020 11:04:51 +0100 Subject: [PATCH 13/22] Added a simpler testpackage --- docker/run-tests.sh | 2 +- .../python-vdt.simpleaptrepo_0.0.3_all.deb | Bin 6998 -> 0 bytes .../tests/testdata/testpackage_1.0_amd64.deb | Bin 0 -> 1068 bytes 3 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 vdt/simpleaptrepo/tests/testdata/python-vdt.simpleaptrepo_0.0.3_all.deb create mode 100644 vdt/simpleaptrepo/tests/testdata/testpackage_1.0_amd64.deb diff --git a/docker/run-tests.sh b/docker/run-tests.sh index 0285dd3..387cc2a 100755 --- a/docker/run-tests.sh +++ b/docker/run-tests.sh @@ -16,7 +16,7 @@ mkdir /repos # now we really create a repo here and add a package # the gpg key is build into the docker image for testing purposes -simpleapt create-repo myrepo /repos --gpgkey AC46C6AAD5792E7E +simpleapt create-repo myrepo /repos --gpgkey B7C72A100F81017B simpleapt add-component myrepo test simpleapt list-repos cp /usr/local/src/vdt/simpleaptrepo/tests/testdata/*.deb /repos/myrepo/test diff --git a/vdt/simpleaptrepo/tests/testdata/python-vdt.simpleaptrepo_0.0.3_all.deb b/vdt/simpleaptrepo/tests/testdata/python-vdt.simpleaptrepo_0.0.3_all.deb deleted file mode 100644 index 0a5087d9c6549d4be0bafeb126b7aa03627612ef..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6998 zcmZ{obxa&kx9ySQ?(SBgxWiDaP@H0g;_k(@xD_4Twdf$FxVyW%41+^)cV=#L@5`6> zlKW0hR(4LZ*B^T&znvU9fT^pw%|~<#OEX(jM-DSvM^jfHT3T8j0UjQ1egSR~VOm-N zZk~THyxjkUmWP{Lke{EH_TT+K#{`|1lN;UK$2KGRvFv!xmfDb>Qh!f0ELdByqd#=T=%*Dj6dbCi=+!vQhyR`v`HK0B1H~P3kA~*doq&$%ZZZRG}}^ z{W7l|`{tc-s6?Za&CCZD?QSW1ts|%Iw?T$nh)^f`v-FPIQ+=?#Vi{$ACMHujjUwdi1e)S|i@I68|@YDOgFz7z`B@GZ)6(4RGYNArmHQcNYU`V zEK8%rj{8qq2ZFTVZbEa{=!4@}JzH}?tZjF^%CaQ^p#fE6aw?xM<`Ic4T2O`G?iB^&*P3Y%h=G7_vhtELpidWxv?kehJ@QT77& zpV@W1(m{l=E*&}=!W(Z*J3@&c!OW{A4Jrj;Qs3>(E9Z!QCt_!mBXL#k$SG!;tqP+F zseqje*xrvT@*{x^Qh;t>`L(#GZX3h$<4gXYGb+4r#HsYJ&=mdLVcTvoSKcSwl`b61 zb>t^4N%`7f6*tywF=o=L?C)#{-8tc!2SEa^0NZ*?ThWDVb}=91$UwR3X=$WQC-!XT zqOqT+jRPK@l8ZicMJs$03Sr-?OogMH6;rmf-|#3^ms5D2l0Q2~4L51R0OZ9DQ;DUQ z90>3)3D*5iZHQeQBcHNG(mwmsj8ol@ouTbW;vYfQlhroof)9 z3CG~NV{vyzs8N4pvWbi6jpH44TSnKtzo;Dwnl@%Lfsg>lZ_f_GskVyu7~g_4L(08| zM0>a>BzmY$L_OaM!wGD>8r;z(W&2Ak;E3eZq9u%6dsqZmf2fK5xrJ*zQx4Fc%FIs1 zRI004hOFn*q7TVFPM(0CgHLg^7t1c)$C4R zb>+!!M)-abdJkY!f`oHW#aY|qQIAKM2S2DrCV0oIdmZzs-GJxf6X;sm-@pHp3**8ZHiYJV4BI|DOH2Zc!|&<~(U zzXm1w(K6yn;Ih)Z1P?GMGH#)~laAg(L_$MLxX*VxFEZk6Kcj9=pIDfkYBKb=Xg$nx zJ8wPAd8t`z@!X$zcjPI0$|ZTv^|G;ME99jwCI(ie$h&CFsk-$EyRD zLM1}hyiYZ!yEQm;jiF%Adc%=G3fg^%+0Gun;Vyh}U43ZP!REOU9qha0SLku(U!(#@ z^%nyFJ?0zf10q@QSColizp1ElVu*Uhueo$hOl2moKpWG=GVc6@XkS3tet2Xw_)U z@yB;jWKrJ<08hDkHA5oeNO0rF)SwewSmM^w8}oQ3SGhBMWoiTZKcTL*S&Fc3YamuUfOX59F`tO$jDpc}(CO7yWxeAiro?o_O|7W=v=cjT*+gQzP3 zT|a4xesp5**>}o(V_yiq+lPR`><6$a$q#{!yX$sdyID0|uw}ao!fOcn-e{o7vlMs{ zsOkSnDDmCDaKyl1$A2;O3|^z??2~zH;a?eBdo?G(1Wg2b@G5n(3xx83OJ-LV}6HXkBc$jnyZdkCbtS#B{ zTxh|r3jh2va)Z;cC(wj-(kT8iNn zwdH)TO4?%jD@yP|ByoXM4Dd~68tWK&52KA}PAu=x`_ls&u-2#TuO=trT`f*g9Fazu zXu-#T@Vt6Ch6cqa6;Xmun2|f0inm>Ea%h*S@5~j4+9dv@Rk|(4<*?48ARbZGtq36w zXD61E#eU-$W%kpdD-_y4)<~KXZUm>OiL7mcX^ zXpVyG(oTUxn~YNy{8v2`E7a%HFZ=-vpH{^q>}yQfrm5GWNcR0ux=W0jzwoXQ>a3!f zB8!7C>dd(=wNAeIU&WL|;Lwn8y$gAV1BqSZd z4jrfcfyKc#y8UMGQjI-`X0Je8S_=WoNhku6ycju8L_HoF-8|3!8?z?ZN}TumrNRex z%bf+vbGG;)=Ex}&_odjC6l}@5I4lFV;QCsX@k~^+x>ANM+cz*uaaW`S)-P`xy%fKw z$L*e8m$2mb^A*o*24juv3Rr8uw6|@Nhn!Oy#CyI2C9K)12at0V zLQ49jRhapNN=_tubErrcwK7xMzz8N0iCP+2^wqGn{Jr(XTJH!01%al9`Vp=Ak{ZA$ zcjP0<6Wcgqg!Rasykl*gWTr%K&NL&nd(JPkAJ7MijQ|n0QlixUGt17 zPPg*xv*}rW$TsGzJPJMuPND{_1X3#39UgEG>+4joX2}kh?ayy?r9cqoOGBgTdyPW3 z35qbD@$O~BQ0BuitrhN`uI;oX-CTfCzn7iLEx?}OMD%x$j^bpgA3uwdr5<(*Wg|7V9+C&;+#gnwM8hSX8}AcDIdlkG zEaWo>bQs3zbxbp6bC~r*6O0K4T&wBDQ5a*)vdarg44!+}iB*5bt`W>e{|X^!ZYh5k zH{y{1@AeahR?7UC?2%yx1rUg}6yd($DP|(+A);(Zf0CWKb#0W90l0LFd_TSQb^iTeQJs;_?(;Xxb$ zdYVG+C{clL;RXr$fNKd6a|JBgcl!uk!F@6B}&s8VIvrpUu6fVqZA`>OXhodU`@i53c6%$ zoemia1t(b)_qA^0X0)@~-bMS9fgN_T14pUH_+1k{O^YX#Vx=m_9^h=a$vaUHa$ZTl z&#IN^a4Xxgq6j4y#FRs7BdnKcrUzg6xAv;azsl^i!QFJdR6$Vd%XJr1+Y8>#L>4@GXZv(Og zrkj8g@^b-v3Z@KuzLeHJjV7lI*|56=e--@Q*n12k=J?^-O-eV@!m%UbXBWOc&CKOw zhu@roR|%>)e5Y(n3k8sd9B1jY^7{@+)2T9?ge)yJB`aTgb zk)@0_Cn-B=Et;0I^RAa%i49#W#eSuZD$Sn3%=vMDAGwVmA~La$ji~bwTkDgi`$cnf zlGHxwmYkqjI)V0`3r8W-;1@?SS_im@eo#Pkt&(-z z>t;d(B_2(p-;Q*7H7~2tgqswTd8^G4J|&CS*`!9MGnw=pz&guG5|LUH)|PDZMMGQ& z_tSj(5l|A19)%obXM)MUC5Z)Zuvl6% zS<}0Cufts3TQCOy#>4n)6lsp4#6T7pL zB9{Dq_NX`d^HM%zJafKGs>FjH&|fWOD=a&{3B7O~KVaxpWm}|iIu%>y;(ZR|7ggWZ z4~oAZEbv3A?_@CicUaUkV?^jbhX**bVPzI%Iz|3~jk~F(yXb5w`VRJhHkdjw({UNU z<0^*mMxj=rL~C2#t9Sh2=+Vh7xERa5m{yzTYAIPN7b;PNzABe6T5;Y+ z$V~7-f`!Y7F>U4AQt=Oj^(7V~-|ZwNlTdCgB&I|wnVo29ddQw0N-X7fyiXn#!oOen zdGe~p7)&>*fWURWwGXbNDbYHPXEb1jjVF28GX`<#!q#n@^jGh?msCL`Fh|4#-cHv} z36`5|VfV@YstLg+o91^fFA1yqdCq3OjfInzVNBsq2ibBK-mS8&J~vhf-&=WdRVk9m z*wIWaY0XB_KUVK8lZp|n^)M-Z!+8hbJyX{bV5@=a4Y4b9?^p~~VVMJ88jfS8+zJg5 z0*J9;?}mAA!swezoR`8qB`C4WQRH|P3*g7LgR1GK{8Du^UD9f(kSGhr8Z;Ot?E*Y% z5zvw!)&-|lqExu$Cw8i+%s?^BN^n%yMbS%Ay`1G~e843eB(Vdw5dE9t385d7TKWf* zDNJJC6N0~l4K>%fO-sjsY&WW2Ok56{}$GT5UiZ{0@7SKFg| zI+)#f6*95v-{&iY$4!6qHW zz?l0$0sZ#BH~D9tApKL=8v4LTL*P5U^gf#5?%-@|W|<};^qe@p<0sronGQks5tYKz z1$oeq{z)slnlfZh%Sip`)UVo9c4_8EX|Up*Pzr=|8ml%V({zZI-QqFUrr;5^NfJjP z`~p8zZazg(>*sf}GA$(AJU~@loQjr8zluUunsjk3e{QL^-IvRsJJF-z$_DC_siXMK+;3d;fTA~@zcyN* z+iOlWf*Pk`&Fk%AgKuj3!N5@o^c|}#%Qa9JdCvF=0@p_vWNaj`I`kos_y)EpV|r`s zu(I-~X5f@T`GQIk2RnVs--0%`4?aSW0izp&PYr~lTAm`_2J&zFZ zI{S5?ez4hP_sTzuz}JJ5yY|o3;v=@^PmfLl?YR!G-Sexg=Yi|!8=&BiGl-o~q`bY0 z#I!Xws$jn5><6flI$bG3NwPEVZ0EOlN?apO(Qg)+f=n5ogb+O~Ng(iY?mjOvej+M= zv8<(~*5zhsjU;y;$bpGuvHc%6XJ<)g_=fLz_M@yE?k zFBdhF((On9aUs!)kMw@_phqDNi^wS^<*?%Q9G012)|MlIC)Et0nym`e1;5KOa^aRQ z_rg=@`L(u5?j1}1(34_yW*!X}?M=Q(Myr)mS8+uhqUBxJJ}Ib^`tJ$UyNfWyz720^ za6(DjT-?f!=SncYJR*jS8ln^PU>F76&59M_2Ojw;VHw@9rzhxpyJ;M3mY|WD;pCS~ zv}Owx`lDY}y!f591Y%PYW+(k!;UChV@bk=$4nY1<^-#6gefnLuIP=ddkBE>oL;UtD^$TJPpsU z7nxcZM<+b@jak*;=ZUKI2pfp13oKG?=8wuRf}Z@I&{ZJzb-N3~Tu&bDm2rx9*T|1A zU>RtZ~G8Y%0 zLBY<`@@KBV2k|C^4 zLpma^1JpYn6$8zg2j1bzt0u(-fF}-Dhrq_>a@B$KZH^^eDQkCfdA+R$LU;`;8Nt+k z+&(9dagxl1->Kr7f?_PW5f}i_gh>HV@ zbLvH4hciLFNx2t3tjjCJ2%lVGl_+-UW7@>`Sel=2(l8&$pRmjxG5s#N8}<<;Jv!7w zRdO#eIf_NGnU{9fG;bD^zls+iPpG3fUd~ZWY4vAw$N)N0h50*4NstLEA^|*E!?t`4 zlO{`B(gUPaKMQTQI;h3$Fnpr!(cZ-c6Bc1f%N?5rK}@EP>0Vg-J?mTbyQ`Z4*a#F_G>yhmJ>Ak_@Vyd5l+rWqme*`>;JHVXGx59_Iw? z!n6JaHc1a_rh=2;bLR=&Ngr^M1oX1**p7NiPGZgY<23xzW%4#MHQ$Bp_3lPQc&WPn zgrUE{>YrfXT&r4WH&IKVv-?A%7)!jCa}Uj?NQ*x#?l{r{L(F2rF!+^^pDQ4gUN>xYfT3XUDW5}91By7@f=~7DLr|;*R-N!~fqaE-V3bX(JWm&ro{azFH+ulg zi@m1q706>m2W&l^^|fzuNrWwLjy?xYT7iUN%|wr_)&8ezfzUy&Hp-7qD(NZ+v5@J^!khfd)($aftnd=KKnQy<9mhYVF6vUe%z3 putukauBXoGQ|RKKrxzLlbM*JWf)M^&@-!bMbg0y^;C{iu{TH-hxBdVC diff --git a/vdt/simpleaptrepo/tests/testdata/testpackage_1.0_amd64.deb b/vdt/simpleaptrepo/tests/testdata/testpackage_1.0_amd64.deb new file mode 100644 index 0000000000000000000000000000000000000000..7d3de608da24247183d511b1750ff104b7695b10 GIT binary patch literal 1068 zcmY$iNi0gvu;WTeP0CEn(@o0EODw9?S5PoeKmZ6ELYkSFfF&R#h?&4;q-VgDoS#=x zl%Jzll31jdUZqc1zmWyhf&?!4ZVm>c+c)DGnBQKB&%Uc5!tlZ7x8@%vbG5)^A+N}V zt-N6x+HdZzFb|)VVUYIv*URVcS}(rB zle%^-o3nYB*ZJ3*RvDey9v?Mrt@jkh$J;%=>YqBf8(`}Y@qC3*wT>UkDk=^S3 z32V+3Ykp0eFQl*Ti4HpQN^Z{o zcbB4<*S`3`?(4cc+=(;l>IZ$+}oyST5hOV^j&usNo1(cOOL&mA|Mdy3sVDyLkwnHBT1(_(*g z^IhI=vkPC`($Tu{(`4@F%wx~1Yq~1l*RnHUfPc&8bG$X1%b>x)kdj!E2#;i73=&Ky zCfMTn^5ygemIe$=#lgV_yeuzFX5?n%FYe%VaQmB|mtzz4?=rK)qx}}j8#}_b+DCrf-}o`(ufKSM%zw?FCrvz=Q;zJP zc`)Pmd_4x+f0-Ah6>R_YHokIyJYS9>{*QXP%Cn`*pFLUrdTDrI68pk$`zI<^{h2K_ zdshFp|1TMz?6#EX$-Cc^o^;JN^=$14hb3_sp#}>Ytypx?0|i zUUsJA*Jsoh-}xi<6l|IDPLj59M%&c7O@6>q06aPz^B%H2J$ zp0zFgx2ZDy`@EZL12-&xdgb=d{Ja*q+1GE}Q`-GBV}*-!fAWU+M#dXo1ZzLN8-6Y& hTjGU-1nc4ik)EbN11`6N8rV3A91Q;Y%X>g^4**FW+gtzu literal 0 HcmV?d00001 From 65983dd86e7534571296a8d64757e8b255680258 Mon Sep 17 00:00:00 2001 From: Martijn Jacobs Date: Wed, 16 Dec 2020 11:06:58 +0100 Subject: [PATCH 14/22] Added pylint and black to the project --- .travis.yml | 2 +- Makefile | 10 +++++++++- setup.py | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 72bf0f4..3ca4690 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,4 +11,4 @@ install: - make install script: - - make docker-test + - make docker_test diff --git a/Makefile b/Makefile index 40dff9b..7f4c6cb 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,4 @@ +.PHONY: clean nosetest install lint black test docker_test clean_release release_testpypi release clean: find . -name '__pycache__' -delete find . -name '*.egg-info' -delete @@ -8,9 +9,16 @@ nosetest: install: pip install -e .[dev] +lint: + black --check vdt/ + pylint setup.py vdt/ + +black: + black --exclude "migrations/*" vdt/ + test: nosetest -docker-test: +docker_test: docker run --rm -v $(PWD):/usr/local/src maerteijn/simpeapt-test-image ./docker/run-tests.sh clean_release: clean diff --git a/setup.py b/setup.py index 5340f8c..3ea5748 100755 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ }, # mark test target to require extras. extras_require={ - 'dev': ['nose', 'coverage', 'mock', 'twine'], + 'dev': ['nose', 'coverage', 'mock', 'twine', 'black', 'pylint'], }, python_requires='>=3.6', ) From f3d53900835cdff0563bc31baf12402d1e796eb9 Mon Sep 17 00:00:00 2001 From: Martijn Jacobs Date: Wed, 16 Dec 2020 11:07:17 +0100 Subject: [PATCH 15/22] Applied black --- vdt/__init__.py | 2 +- vdt/simpleaptrepo/cli.py | 55 ++++++++++++------------ vdt/simpleaptrepo/config.py | 4 +- vdt/simpleaptrepo/repo.py | 58 ++++++++++++++------------ vdt/simpleaptrepo/tests/test_cli.py | 45 +++++++++----------- vdt/simpleaptrepo/tests/test_config.py | 7 ++-- vdt/simpleaptrepo/tests/test_utils.py | 7 ++-- vdt/simpleaptrepo/utils.py | 2 +- 8 files changed, 88 insertions(+), 92 deletions(-) diff --git a/vdt/__init__.py b/vdt/__init__.py index de40ea7..5284146 100644 --- a/vdt/__init__.py +++ b/vdt/__init__.py @@ -1 +1 @@ -__import__('pkg_resources').declare_namespace(__name__) +__import__("pkg_resources").declare_namespace(__name__) diff --git a/vdt/simpleaptrepo/cli.py b/vdt/simpleaptrepo/cli.py index 063bcb4..a20d337 100644 --- a/vdt/simpleaptrepo/cli.py +++ b/vdt/simpleaptrepo/cli.py @@ -10,7 +10,7 @@ def cli(): pass -@cli.command(name='create-gpg-key') +@cli.command(name="create-gpg-key") def create_key(): """Creates a GPG key""" try: @@ -22,10 +22,10 @@ def create_key(): click.echo("Now add a repository with the 'create-repo' command") -@cli.command(name='create-repo') -@click.argument('name') -@click.argument('path', default=".") -@click.option('--gpgkey', help='The GPG key to sign the packages with') +@cli.command(name="create-repo") +@click.argument("name") +@click.argument("path", default=".") +@click.option("--gpgkey", help="The GPG key to sign the packages with") def create_repo(name, path, gpgkey=""): """Creates a repository""" try: @@ -37,9 +37,9 @@ def create_repo(name, path, gpgkey=""): click.echo("Now add a component with the 'add-component' command") -@cli.command(name='add-component') -@click.argument('name') -@click.argument('component', default="main") +@cli.command(name="add-component") +@click.argument("name") +@click.argument("component", default="main") def add_component(name, component): """Creates a component (ie, 'main', 'production')""" try: @@ -54,28 +54,25 @@ def add_component(name, component): click.echo("and run the 'update-repo' command") click.echo("") click.echo( - "After that, configure your webservice " - "to set the www-root to %s " % root) - click.echo( - "Add http:///%s/%s / to your sources.list" % ( - name, component)) + "After that, configure your webservice " "to set the www-root to %s " % root + ) + click.echo("Add http:///%s/%s / to your sources.list" % (name, component)) click.echo("") - click.echo( - "Add the key on the host where you want to install the packages.") + click.echo("Add the key on the host where you want to install the packages.") click.echo("(This is only needed once per repository)") click.echo( - "wget -qO - http:///%s/%s/keyfile | sudo apt-key add -" % ( - name, component)) + "wget -qO - http:///%s/%s/keyfile | sudo apt-key add -" + % (name, component) + ) -@cli.command(name='update-repo') -@click.argument('name') -@click.argument('component', default="main") -@click.option( - '--skip-signed', is_flag=True, help='Skip already signed packaged') +@cli.command(name="update-repo") +@click.argument("name") +@click.argument("component", default="main") +@click.option("--skip-signed", is_flag=True, help="Skip already signed packaged") def update_repo(name, component, skip_signed=False): """Updates a repo's component by scanning the debian packages - and add the index files. + and add the index files. """ try: repo_cfg = apt_repo.get_repo_config(name) @@ -83,19 +80,20 @@ def update_repo(name, component, skip_signed=False): except ValueError as e: raise click.BadParameter(str(e)) - gpgkey = repo_cfg.get('gpgkey', None) + gpgkey = repo_cfg.get("gpgkey", None) apt_repo.update_component( - component_path, gpgkey, skip_signed, output_command=click.echo) + component_path, gpgkey, skip_signed, output_command=click.echo + ) -@cli.command('list-repos') +@cli.command("list-repos") def list_repos(): """List currently configured repos""" repos = apt_repo.list_repos() for repo in repos: - click.echo(repo.get('name')) - for component in repo.get('components'): + click.echo(repo.get("name")) + for component in repo.get("components"): click.echo(" {0}".format(component)) @@ -105,5 +103,6 @@ def main(): else: click.echo("You are not on debian or ubuntu, aborting!") + if __name__ == "__main__": main() diff --git a/vdt/simpleaptrepo/config.py b/vdt/simpleaptrepo/config.py index 511dbcc..9a2f8ca 100644 --- a/vdt/simpleaptrepo/config.py +++ b/vdt/simpleaptrepo/config.py @@ -19,10 +19,10 @@ def add_repo_config(self, name, path, gpgkey=""): if not self.config.has_section(name): self.config.add_section(name) - self.config.set(name, 'path', path) + self.config.set(name, "path", path) if gpgkey: - self.config.set(name, 'gpgkey', gpgkey) + self.config.set(name, "gpgkey", gpgkey) self.save_config() diff --git a/vdt/simpleaptrepo/repo.py b/vdt/simpleaptrepo/repo.py index 571ed7e..fac7bba 100644 --- a/vdt/simpleaptrepo/repo.py +++ b/vdt/simpleaptrepo/repo.py @@ -16,9 +16,11 @@ def create_gpg_key(output_command): def export_pubkey(path, gpgkey, output_command): - key_path = os.path.join(path, 'keyfile') + key_path = os.path.join(path, "keyfile") cmd = "/usr/bin/gpg --digest-algo SHA256 --yes --output %s --armor --export %s" % ( - key_path, gpgkey) + key_path, + gpgkey, + ) subprocess.check_output(cmd, shell=True) output_command("Exported key %s to %s" % (gpgkey, key_path)) @@ -28,7 +30,8 @@ def sign_packages(path, gpgkey, skip_signed, output_command): for deb_file in glob(os.path.join(path, "*.deb")): try: output = subprocess.check_output( - "dpkg-sig --verify %s" % deb_file, shell=True) + "dpkg-sig --verify %s" % deb_file, shell=True + ) except subprocess.CalledProcessError as e: # this is pretty strange, as the command is returning an error code # but still could have run succesful @@ -43,47 +46,50 @@ def sign_packages(path, gpgkey, skip_signed, output_command): output_command("Package %s already signed!" % deb_file) output_command("Removing signature") - subprocess.check_output( - "ar -d %s _gpgbuilder" % deb_file, shell=True) + subprocess.check_output("ar -d %s _gpgbuilder" % deb_file, shell=True) # we should match the gpgbuilder output with a regex, but for now # we do this for a quickfix. The output can be _gpgbuilder or # _gpgbuilder0, probably _gpgbuilder1 etc too - subprocess.check_output( - "ar -d %s _gpgbuilder0" % deb_file, shell=True) + subprocess.check_output("ar -d %s _gpgbuilder0" % deb_file, shell=True) # sign again output_command("Signed package %s" % deb_file) subprocess.check_output( - "/usr/bin/dpkg-sig -k %s --sign builder %s" % ( - gpgkey, deb_file), shell=True) + "/usr/bin/dpkg-sig -k %s --sign builder %s" % (gpgkey, deb_file), shell=True + ) def create_package_index(path, output_command): output_command("Creates Packages") subprocess.check_output( - "/usr/bin/apt-ftparchive packages . > Packages", shell=True, cwd=path) + "/usr/bin/apt-ftparchive packages . > Packages", shell=True, cwd=path + ) output_command("Creates Packages.gz") - subprocess.check_output( - "/bin/gzip -c Packages > Packages.gz", shell=True, cwd=path) + subprocess.check_output("/bin/gzip -c Packages > Packages.gz", shell=True, cwd=path) def create_signed_releases_index(path, gpgkey, output_command): output_command("Create Release with key %s" % gpgkey) subprocess.check_output( - "/usr/bin/apt-ftparchive release . > Release", shell=True, cwd=path) + "/usr/bin/apt-ftparchive release . > Release", shell=True, cwd=path + ) output_command("Create InRelease with key %s" % gpgkey) subprocess.check_output( - "/usr/bin/gpg --digest-algo SHA256 --yes -u 0x%s --clearsign -o InRelease Release" % ( - gpgkey), shell=True, cwd=path) + "/usr/bin/gpg --digest-algo SHA256 --yes -u 0x%s --clearsign -o InRelease Release" + % (gpgkey), + shell=True, + cwd=path, + ) output_command("Create Releases.gpg with key %s" % gpgkey) subprocess.check_output( - "/usr/bin/gpg --yes -u 0x%s -abs -o Release.gpg Release" % ( - gpgkey), shell=True, cwd=path) + "/usr/bin/gpg --yes -u 0x%s -abs -o Release.gpg Release" % (gpgkey), + shell=True, + cwd=path, + ) class SimpleAPTRepo(Config): - def add_repo(self, name, path, gpgkey=""): if not os.path.exists(path): raise ValueError("Path does not exists!") @@ -99,7 +105,7 @@ def add_repo(self, name, path, gpgkey=""): def add_component(self, name, component): repo_cfg = self.get_repo_config(name) - path = os.path.join(repo_cfg['path'], component) + path = os.path.join(repo_cfg["path"], component) if os.path.exists(path): raise ValueError("Directory %s already exists!" % path) os.mkdir(path) @@ -108,7 +114,7 @@ def add_component(self, name, component): def get_component_path(self, name, component): repo_cfg = self.get_repo_config(name) - path = os.path.join(repo_cfg.get('path'), component) + path = os.path.join(repo_cfg.get("path"), component) if not os.path.exists(path): raise ValueError("Component '%s' does not exist!" % component) return path @@ -118,16 +124,16 @@ def list_repos(self): for section in self.config.sections(): repo = {} repo_cfg = self.get_repo_config(section) - if repo_cfg.get('gpgkey'): - section = "%s (gpgkey: %s)" % (section, repo_cfg.get('gpgkey')) - repo['name'] = section - repo['components'] = os.listdir(repo_cfg.get('path')) + if repo_cfg.get("gpgkey"): + section = "%s (gpgkey: %s)" % (section, repo_cfg.get("gpgkey")) + repo["name"] = section + repo["components"] = os.listdir(repo_cfg.get("path")) result.append(repo) return result def update_component( - self, path, gpgkey=None, skip_signed=False, - output_command=write_to_stdout): + self, path, gpgkey=None, skip_signed=False, output_command=write_to_stdout + ): if gpgkey is not None: # export keyfile export_pubkey(path, gpgkey, output_command) diff --git a/vdt/simpleaptrepo/tests/test_cli.py b/vdt/simpleaptrepo/tests/test_cli.py index e95dffb..ca62cff 100644 --- a/vdt/simpleaptrepo/tests/test_cli.py +++ b/vdt/simpleaptrepo/tests/test_cli.py @@ -48,7 +48,6 @@ class TestCLI(unittest.TestCase): - def setUp(self): # patch the aptrepo instance to write the file to the current # directory, re-read the sections @@ -69,7 +68,7 @@ def tearDown(self): cli.apt_repo.config = ConfigParser.ConfigParser() - @mock.patch('subprocess.check_output', return_value="GPG OUTPUT") + @mock.patch("subprocess.check_output", return_value="GPG OUTPUT") def test_create_gpg_key(self, mock_subprocess): runner = CliRunner() result = runner.invoke(cli.create_key) @@ -81,7 +80,7 @@ def test_create_gpg_key(self, mock_subprocess): # click's output should be captured correctly self.assertEqual(result.output, GPG_KEY_OUTPUT) - @mock.patch('subprocess.check_output', side_effect=ValueError('GPG ERROR')) + @mock.patch("subprocess.check_output", side_effect=ValueError("GPG ERROR")) def test_create_gpg_key_exception(self, mock_subprocess): runner = CliRunner() result = runner.invoke(cli.create_key) @@ -97,8 +96,7 @@ def test_create_repo_and_component(self): runner = CliRunner() with runner.isolated_filesystem(): - result = runner.invoke( - cli.create_repo, ["my_repo", "--gpgkey", "123456"]) + result = runner.invoke(cli.create_repo, ["my_repo", "--gpgkey", "123456"]) # command should be run without error self.assertEqual(result.exit_code, 0) @@ -113,19 +111,19 @@ def test_create_repo_and_component(self): self.assertTrue(os.path.exists(cli.apt_repo.path)) # now create a component - result = runner.invoke( - cli.add_component, ["my_repo", "main"]) + result = runner.invoke(cli.add_component, ["my_repo", "main"]) # command should be run without error self.assertEqual(result.exit_code, 0) # click's output should be captured correctly self.assertTrue( - "Add http:///my_repo/main / to your sources.list" in result.output) # noqa + "Add http:///my_repo/main / to your sources.list" + in result.output + ) # noqa # adding the same component should raise an error - result = runner.invoke( - cli.add_component, ["my_repo", "main"]) + result = runner.invoke(cli.add_component, ["my_repo", "main"]) self.assertEqual(result.exit_code, 2) self.assertTrue("already exists!" in result.output) @@ -137,21 +135,21 @@ def test_create_repo_and_component(self): self.assertTrue(os.path.exists("another_repo")) # now create a component - result = runner.invoke( - cli.add_component, ["another_repo", "main"]) + result = runner.invoke(cli.add_component, ["another_repo", "main"]) # command should be run without error self.assertEqual(result.exit_code, 0) # click's output should be captured correctly self.assertTrue( - "Add http:///another_repo/main / to your sources.list" in result.output) # noqa + "Add http:///another_repo/main / to your sources.list" + in result.output + ) # noqa def test_create_repo_unkown_path(self): runner = CliRunner() with runner.isolated_filesystem(): - result = runner.invoke( - cli.create_repo, ["my_repo", "/unknown/path/"]) + result = runner.invoke(cli.create_repo, ["my_repo", "/unknown/path/"]) # there is an error as the path is unknown self.assertEqual(result.exit_code, 2) @@ -172,8 +170,7 @@ def test_list_repos(self): # create a repo and a component with runner.isolated_filesystem(): - result = runner.invoke( - cli.create_repo, ["my_repo", "--gpgkey", "123456"]) + result = runner.invoke(cli.create_repo, ["my_repo", "--gpgkey", "123456"]) self.assertEqual(result.exit_code, 0) result = runner.invoke(cli.add_component, ["my_repo", "main"]) self.assertEqual(result.exit_code, 0) @@ -183,28 +180,24 @@ def test_list_repos(self): self.assertEqual(result.exit_code, 0) self.assertEqual(result.output, LIST_REPOS_OUTPUT) - @mock.patch('subprocess.check_output', return_value="Mocked output") + @mock.patch("subprocess.check_output", return_value="Mocked output") def test_update_repo(self, mock_subprocess=None): runner = CliRunner() with runner.isolated_filesystem(): # create a repo and a component - result = runner.invoke( - cli.create_repo, ["my_repo", "--gpgkey", "123456"]) + result = runner.invoke(cli.create_repo, ["my_repo", "--gpgkey", "123456"]) self.assertEqual(result.exit_code, 0) result = runner.invoke(cli.add_component, ["my_repo", "main"]) self.assertEqual(result.exit_code, 0) # we mock everything, however, the runner still should say that # everything is created / updated - result = runner.invoke( - cli.update_repo, ["my_repo", "main"]) + result = runner.invoke(cli.update_repo, ["my_repo", "main"]) self.assertEqual(result.exit_code, 0) self.assertIn(UPDATE_REPO_OUTPUT, result.output) # Let's update an component which does not exist - result = runner.invoke( - cli.update_repo, ["my_repo", "i-am-not-there"]) + result = runner.invoke(cli.update_repo, ["my_repo", "i-am-not-there"]) self.assertEqual(result.exit_code, 2) - self.assertIn( - "Component 'i-am-not-there' does not exist!", result.output) + self.assertIn("Component 'i-am-not-there' does not exist!", result.output) diff --git a/vdt/simpleaptrepo/tests/test_config.py b/vdt/simpleaptrepo/tests/test_config.py index 8bd05cf..2bdc61f 100644 --- a/vdt/simpleaptrepo/tests/test_config.py +++ b/vdt/simpleaptrepo/tests/test_config.py @@ -5,7 +5,6 @@ class TestConfig(unittest.TestCase): - def setUp(self): config.HOME = os.getcwd() self.configfile = os.path.join(os.getcwd(), ".simpleapt.ini") @@ -26,7 +25,7 @@ def test_add_repo_config(self): config_obj.add_repo_config(name="test", path="/www", gpgkey="123456") # check the values of path and gpgkey - section = config_obj.config.items('test') + section = config_obj.config.items("test") self.assertEqual(section[0][1], "/www") self.assertEqual(section[1][1], "123456") @@ -36,8 +35,8 @@ def test_get_repo_config(self): # this should be fine cfg = config_obj.get_repo_config(name="test") - self.assertTrue('path' in cfg) - self.assertTrue('gpgkey' in cfg) + self.assertTrue("path" in cfg) + self.assertTrue("gpgkey" in cfg) # a unkown name should raise an exception self.assertRaises(ValueError, config_obj.get_repo_config, "unkown") diff --git a/vdt/simpleaptrepo/tests/test_utils.py b/vdt/simpleaptrepo/tests/test_utils.py index 59204b2..1409866 100644 --- a/vdt/simpleaptrepo/tests/test_utils.py +++ b/vdt/simpleaptrepo/tests/test_utils.py @@ -4,19 +4,18 @@ def mock_dist_osx(): - return ('Mac OSX', '10.10.5', 'Yosemity') + return ("Mac OSX", "10.10.5", "Yosemity") def mock_dist_ubuntu(): - return ('Ubuntu', '12.04', 'precise') + return ("Ubuntu", "12.04", "precise") def mock_dist_debian(): - return ('debian', '8.6', '') + return ("debian", "8.6", "") class TestUtils(unittest.TestCase): - def test_platform_is_debian(self): utils.platform.dist = mock_dist_osx self.assertFalse(utils.platform_is_debian()) diff --git a/vdt/simpleaptrepo/utils.py b/vdt/simpleaptrepo/utils.py index a7fce9c..9a34021 100644 --- a/vdt/simpleaptrepo/utils.py +++ b/vdt/simpleaptrepo/utils.py @@ -21,7 +21,7 @@ def repo_root(path): def write_to_stdout(message): - unbuffered_output = os.fdopen(sys.stdout.fileno(), 'w', 0) + unbuffered_output = os.fdopen(sys.stdout.fileno(), "w", 0) unbuffered_output.write("%s\n" % message) From 61f67b20d0e3b5ef33954e90cebd46e39e8270c2 Mon Sep 17 00:00:00 2001 From: Martijn Jacobs Date: Wed, 16 Dec 2020 11:24:58 +0100 Subject: [PATCH 16/22] Applied pylint and added a make lint command --- .travis.yml | 2 ++ setup.cfg | 5 +++++ vdt/simpleaptrepo/repo.py | 2 +- vdt/simpleaptrepo/tests/test_cli.py | 7 ++----- vdt/simpleaptrepo/tests/test_utils.py | 12 ++++++------ vdt/simpleaptrepo/utils.py | 4 ++-- 6 files changed, 18 insertions(+), 14 deletions(-) create mode 100644 setup.cfg diff --git a/.travis.yml b/.travis.yml index 3ca4690..82a7b2d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,4 +11,6 @@ install: - make install script: + - make lint + - make nosetest - make docker_test diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..906400a --- /dev/null +++ b/setup.cfg @@ -0,0 +1,5 @@ +[pylint.MASTER] +jobs=4 + +[pylint.'MESSAGES CONTROL'] +disable = R,C,W5103 diff --git a/vdt/simpleaptrepo/repo.py b/vdt/simpleaptrepo/repo.py index fac7bba..d6dc2bf 100644 --- a/vdt/simpleaptrepo/repo.py +++ b/vdt/simpleaptrepo/repo.py @@ -12,7 +12,7 @@ def create_gpg_key(output_command): try: output_command(subprocess.check_output(cmd, shell=True)) except subprocess.CalledProcessError as e: - raise ValueError(e) + raise ValueError from e def export_pubkey(path, gpgkey, output_command): diff --git a/vdt/simpleaptrepo/tests/test_cli.py b/vdt/simpleaptrepo/tests/test_cli.py index ca62cff..20339a3 100644 --- a/vdt/simpleaptrepo/tests/test_cli.py +++ b/vdt/simpleaptrepo/tests/test_cli.py @@ -1,7 +1,4 @@ -try: - import ConfigParser # python 2 -except: - import configparser as ConfigParser # python 3 +import configparser as ConfigParser import os import shutil @@ -181,7 +178,7 @@ def test_list_repos(self): self.assertEqual(result.output, LIST_REPOS_OUTPUT) @mock.patch("subprocess.check_output", return_value="Mocked output") - def test_update_repo(self, mock_subprocess=None): + def test_update_repo(self, mock_subprocess=None): # pylint: disable=W0613 runner = CliRunner() with runner.isolated_filesystem(): diff --git a/vdt/simpleaptrepo/tests/test_utils.py b/vdt/simpleaptrepo/tests/test_utils.py index 1409866..95d29bc 100644 --- a/vdt/simpleaptrepo/tests/test_utils.py +++ b/vdt/simpleaptrepo/tests/test_utils.py @@ -4,24 +4,24 @@ def mock_dist_osx(): - return ("Mac OSX", "10.10.5", "Yosemity") + return "macOS-10.14.6-x86_64-i386-64bit" def mock_dist_ubuntu(): - return ("Ubuntu", "12.04", "precise") + return "Linux-4.10.0-40-generic-x86_64-with-Ubuntu-16.04-xenial" def mock_dist_debian(): - return ("debian", "8.6", "") + return "Linux-4.10.0-40-generic-x86_64-with-Debian-10" class TestUtils(unittest.TestCase): def test_platform_is_debian(self): - utils.platform.dist = mock_dist_osx + utils.platform.platform = mock_dist_osx self.assertFalse(utils.platform_is_debian()) - utils.platform.dist = mock_dist_ubuntu + utils.platform.platform = mock_dist_ubuntu self.assertTrue(utils.platform_is_debian()) - utils.platform.dist = mock_dist_debian + utils.platform.platform = mock_dist_debian self.assertTrue(utils.platform_is_debian()) diff --git a/vdt/simpleaptrepo/utils.py b/vdt/simpleaptrepo/utils.py index 9a34021..5dd10d8 100644 --- a/vdt/simpleaptrepo/utils.py +++ b/vdt/simpleaptrepo/utils.py @@ -26,5 +26,5 @@ def write_to_stdout(message): def platform_is_debian(): - current_platform = platform.dist()[0].lower() - return current_platform in ["ubuntu", "debian"] + current_platform = platform.platform().lower() + return "ubuntu" in current_platform or "debian" in current_platform From a955cf8daf39bbff37c2aa0796140bdb6ef63fde Mon Sep 17 00:00:00 2001 From: Martijn Jacobs Date: Sun, 20 Dec 2020 11:23:11 +0100 Subject: [PATCH 17/22] Readded python3.5, still the default python on many debian bases distros --- .travis.yml | 1 + README.rst | 6 +++--- setup.py | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 82a7b2d..ee60345 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,6 @@ language: python python: + - "3.5" - "3.6" - "3.7" - "3.8" diff --git a/README.rst b/README.rst index 416cf22..cc0f877 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ vdt.simpleaptrepo ================= -Simple command line tool to create apt repositories. This will work on debian and ubuntu. +Simple command line tool to create apt repositories. This will work on debian and ubuntu. Requires `python>=3.5` .. image:: https://api.travis-ci.org/devopsconsulting/vdt.simpleaptrepo.svg?branch=master :target: https://travis-ci.org/devopsconsulting/vdt.simpleaptrepo @@ -58,7 +58,7 @@ You will see what you need to do now:: Configure your webservice to set the www-root to /www/ Add http:///myrepo/test / to your sources.list - + Add the key on the host where you want to install the packages. (This is only needed once per repository) wget -qO - http:///myrepo/test/keyfile | sudo apt-key add - @@ -71,7 +71,7 @@ Add some more if you like:: See that our repo is there:: simpleapt list-repos - + myrepo (gpgkey: 10FB8BDC) test staging diff --git a/setup.py b/setup.py index 3ea5748..f1ff6a8 100755 --- a/setup.py +++ b/setup.py @@ -15,6 +15,7 @@ 'Operating System :: Unix', 'Programming Language :: Python', 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', @@ -47,5 +48,5 @@ extras_require={ 'dev': ['nose', 'coverage', 'mock', 'twine', 'black', 'pylint'], }, - python_requires='>=3.6', + python_requires='>=3.5', ) From 1657236b9faebdcb7a4c0435a4d264de9dc4da16 Mon Sep 17 00:00:00 2001 From: Martijn Jacobs Date: Tue, 12 Jan 2021 10:54:45 +0100 Subject: [PATCH 18/22] First attempt to use github actions for this --- .github/workflows/ci.yml | 16 ++++++++++++++++ .travis.yml | 17 ----------------- 2 files changed, 16 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/ci.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..7c829ec --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,16 @@ +name: vdt.simpleaptrepo ci +on: [push] +jobs: + run-ci-steps: + runs-on: ubuntu-latest + steps: + - name: Checkout the repository + uses: actions/checkout@v2 + - name: Setup Python 3.6 + uses: actions/setup-python@v2 + with: + python-version: '3.6' + - name: Install all dependencies + run: make install + - name: Run all linting + run: make lint diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index ee60345..0000000 --- a/.travis.yml +++ /dev/null @@ -1,17 +0,0 @@ -language: python -python: - - "3.5" - - "3.6" - - "3.7" - - "3.8" - -services: - - docker - -install: - - make install - -script: - - make lint - - make nosetest - - make docker_test From 4c71a9f74b5643a3926084df53e910e27a3b8d9a Mon Sep 17 00:00:00 2001 From: Martijn Jacobs Date: Tue, 12 Jan 2021 11:03:32 +0100 Subject: [PATCH 19/22] Added the tests --- .github/workflows/ci.yml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c829ec..aad4e03 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,7 @@ name: vdt.simpleaptrepo ci on: [push] jobs: - run-ci-steps: + run-linting: runs-on: ubuntu-latest steps: - name: Checkout the repository @@ -14,3 +14,19 @@ jobs: run: make install - name: Run all linting run: make lint + + run-tests: + needs: run-linting + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.5, 3.6, 3.7, 3.8, 3.9] + steps: + - name: Setup Python 3.x + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install all dependencies + run: make install + - name: Run the unittests + run: make test \ No newline at end of file From 30ca457327e215e872ef6b77147abddfae59769c Mon Sep 17 00:00:00 2001 From: Martijn Jacobs Date: Tue, 12 Jan 2021 17:07:58 +0100 Subject: [PATCH 20/22] Setup the repository first --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aad4e03..a82a3b8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,8 @@ jobs: matrix: python-version: [3.5, 3.6, 3.7, 3.8, 3.9] steps: + - name: Checkout the repository + uses: actions/checkout@v2 - name: Setup Python 3.x uses: actions/setup-python@v2 with: From 0aba8ef333e19a332dec1e409a5e6acbb0dbec09 Mon Sep 17 00:00:00 2001 From: Martijn Jacobs Date: Wed, 13 Jan 2021 11:49:07 +0100 Subject: [PATCH 21/22] Split make install with make install-dev and make install-lint --- .github/workflows/ci.yml | 2 +- Makefile | 3 +++ setup.py | 3 ++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a82a3b8..ae20d72 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,6 +29,6 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install all dependencies - run: make install + run: make install-dev - name: Run the unittests run: make test \ No newline at end of file diff --git a/Makefile b/Makefile index 7f4c6cb..9ea330b 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,9 @@ nosetest: nosetests -s --with-coverage --cover-erase --cover-package=vdt.simpleaptrepo --cover-xml --logging-level=INFO --with-doctest --verbosity=2 install: + pip install -e .[dev,lint] + +install-dev: pip install -e .[dev] lint: diff --git a/setup.py b/setup.py index f1ff6a8..3fff591 100755 --- a/setup.py +++ b/setup.py @@ -46,7 +46,8 @@ }, # mark test target to require extras. extras_require={ - 'dev': ['nose', 'coverage', 'mock', 'twine', 'black', 'pylint'], + 'dev': ['nose', 'coverage', 'mock', 'twine'], + 'lint': ['black', 'pylint'], }, python_requires='>=3.5', ) From 14743dab3db43832e85a28571426cb67e6f53f9e Mon Sep 17 00:00:00 2001 From: Martijn Jacobs Date: Tue, 19 Jan 2021 17:00:18 +0100 Subject: [PATCH 22/22] Run a separate test script to create a repo with a signed package --- .github/workflows/ci.yml | 40 +++++++++++++++++++++++++++++++++++--- docker/create-test-repo.sh | 33 +++++++++++++++++++++++++++++++ docker/run-tests.sh | 18 +---------------- 3 files changed, 71 insertions(+), 20 deletions(-) create mode 100755 docker/create-test-repo.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ae20d72..177af09 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,11 +1,18 @@ name: vdt.simpleaptrepo ci on: [push] + +defaults: + run: + working-directory: ./src + jobs: run-linting: runs-on: ubuntu-latest steps: - name: Checkout the repository uses: actions/checkout@v2 + with: + path: ./src - name: Setup Python 3.6 uses: actions/setup-python@v2 with: @@ -14,6 +21,11 @@ jobs: run: make install - name: Run all linting run: make lint + - name: Upload src dir as artefact + uses: actions/upload-artifact@v2 + with: + name: src + path: ./src run-tests: needs: run-linting @@ -22,8 +34,11 @@ jobs: matrix: python-version: [3.5, 3.6, 3.7, 3.8, 3.9] steps: - - name: Checkout the repository - uses: actions/checkout@v2 + - name: Download src dir + uses: actions/download-artifact@v2 + with: + name: src + path: ./src - name: Setup Python 3.x uses: actions/setup-python@v2 with: @@ -31,4 +46,23 @@ jobs: - name: Install all dependencies run: make install-dev - name: Run the unittests - run: make test \ No newline at end of file + run: make test + + run-docker: + needs: run-tests + runs-on: ubuntu-latest + defaults: + run: + working-directory: /usr/local/src + container: + image: maerteijn/simpeapt-test-image:latest + steps: + - name: Download src dir + uses: actions/download-artifact@v2 + with: + name: src + path: /usr/local/src + - name: Copy GPG keys to github home directory + run: cp -R /root/.gnupg ~ + - name: Create a test repo and install a signed package + run: bash ./docker/create-test-repo.sh diff --git a/docker/create-test-repo.sh b/docker/create-test-repo.sh new file mode 100755 index 0000000..11680c4 --- /dev/null +++ b/docker/create-test-repo.sh @@ -0,0 +1,33 @@ +#!/bin/bash +export LC_ALL=C.UTF-8 +export LANG=C.UTF-8 + +# install simpleaptrepo +pip3 install -e . + +# do some cleanup +if [ -d "/repos/" ]; then + rm -Rf /repos/ +fi + +# create a repo directory +mkdir /repos + +# now we really create a repo here and add a package +# the gpg key is build into the docker image for testing purposes +simpleapt create-repo myrepo /repos --gpgkey B7C72A100F81017B +simpleapt add-component myrepo test +simpleapt list-repos +cp /usr/local/src/vdt/simpleaptrepo/tests/testdata/*.deb /repos/myrepo/test +simpleapt update-repo myrepo test +simpleapt update-repo myrepo test --skip-signed + +# add the created repo as a local source +echo "deb file:/repos/myrepo/test /" > /etc/apt/sources.list.d/local.list + +# add the GPG key as a valid key: +cat /repos/myrepo/test/keyfile | apt-key add - + +# update the apt repository and install the signed package +apt-get update +apt-get install --yes testpackage \ No newline at end of file diff --git a/docker/run-tests.sh b/docker/run-tests.sh index 387cc2a..82e9987 100755 --- a/docker/run-tests.sh +++ b/docker/run-tests.sh @@ -3,22 +3,6 @@ export LC_ALL=C.UTF-8 export LANG=C.UTF-8 # runs the unittests -pip3 install pip --upgrade +python3 -m pip install pip --upgrade pip3 install -e .[dev] make nosetest - -# do some cleanup -if [ -d "/repos/" ]; then - rm -Rf /repos/ -fi - -mkdir /repos - -# now we really create a repo here and add a package -# the gpg key is build into the docker image for testing purposes -simpleapt create-repo myrepo /repos --gpgkey B7C72A100F81017B -simpleapt add-component myrepo test -simpleapt list-repos -cp /usr/local/src/vdt/simpleaptrepo/tests/testdata/*.deb /repos/myrepo/test -simpleapt update-repo myrepo test -simpleapt update-repo myrepo test --skip-signed