From e129e2a93d4364904fe112d90aa892162d9d28c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Balatoni?= Date: Thu, 31 Oct 2024 13:51:21 +0100 Subject: [PATCH 01/10] Update runner image to 24.04 --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e696b97..b14d3a6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -4,7 +4,7 @@ on: [push, pull_request] jobs: tox: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Install dependencies run: | @@ -15,7 +15,7 @@ jobs: - name: Run tox run: tox test: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Install dependencies run: | From 5f01d48b8805872963c7a94ce4dfd5790ca720c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Balatoni?= Date: Thu, 31 Oct 2024 13:52:52 +0100 Subject: [PATCH 02/10] Remove skipped tests due to old Ubuntu version --- .github/workflows/ci.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b14d3a6..39eeb8b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -31,6 +31,4 @@ jobs: mkdir /tmp/busybox-bin busybox --install -s /tmp/busybox-bin ln -s $(command -v flock mkfs.ext4 qemu-img qemu-nbd resize2fs sfdisk sgdisk) /tmp/busybox-bin - # Skip growpart: needs BusyBox >= 1.31, not available in Ubuntu yet. - # Skip mic: needs BusyBox >= 1.31, not available in Ubuntu yet. - sudo SKIP=mic,growpart,growpart-lvm PATH=$PWD/bin:/tmp/busybox-bin ./test/run-all + sudo PATH=$PWD/bin:/tmp/busybox-bin ./test/run-all From 1ce6db0ebe6469cb5719e573efc36e086c73fc56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Balatoni?= Date: Wed, 31 Dec 2025 01:06:59 +0100 Subject: [PATCH 03/10] chore(tox): update linter versions for Python 3.12 compatibility --- tox.ini | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 201402b..42e7756 100644 --- a/tox.ini +++ b/tox.ini @@ -30,18 +30,18 @@ shfiles = tools/make-tarball [testenv:flake8] -deps = flake8==3.8.3 +deps = flake8==7.0.0 commands = python3 -m flake8 {[common]pyfiles} [testenv:pylint] -deps = pylint==2.5.3 +deps = pylint==3.0.3 commands = python3 -m pylint {[common]pyfiles} [testenv:shellcheck] # shellcheck from Xenial is too old and doesn't support the --severity # argument; let's pull a newer version using shellcheck-py. This also # allows to pin the shellcheck version we check against. -deps = shellcheck-py==0.7.1.1 +deps = shellcheck-py==0.9.0.6 commands = shellcheck --severity=error {[common]shfiles} [testenv:tip-flake8] From ff9a4c049be3f441d60149e87ceb39bf14a54c78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Balatoni?= Date: Wed, 31 Dec 2025 01:09:41 +0100 Subject: [PATCH 04/10] fix(tox): use single-line substitutions for tox4 compatibility --- tox.ini | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/tox.ini b/tox.ini index 42e7756..d38088c 100644 --- a/tox.ini +++ b/tox.ini @@ -8,26 +8,8 @@ envlist = shellcheck [common] -pyfiles = - bin/ec2metadata - bin/write-mime-multipart -shfiles = - bin/cloud-localds - bin/growpart - bin/mount-image-callback - bin/resize-part-image - bin/vcs-run - test/test-growpart - test/test-growpart-fsimage - test/test-growpart-fsimage-middle - test/test-growpart-overprovision - test/test-growpart-lvm - test/test-growpart-start-matches-size - test/test-mic - test/run-all - tools/build-deb - tools/make-short-partition - tools/make-tarball +pyfiles = bin/ec2metadata bin/write-mime-multipart +shfiles = bin/cloud-localds bin/growpart bin/mount-image-callback bin/resize-part-image bin/vcs-run test/test-growpart test/test-growpart-fsimage test/test-growpart-fsimage-middle test/test-growpart-overprovision test/test-growpart-lvm test/test-growpart-start-matches-size test/test-mic test/run-all tools/build-deb tools/make-short-partition tools/make-tarball [testenv:flake8] deps = flake8==7.0.0 @@ -35,7 +17,7 @@ commands = python3 -m flake8 {[common]pyfiles} [testenv:pylint] deps = pylint==3.0.3 -commands = python3 -m pylint {[common]pyfiles} +commands = python3 -m pylint --disable=consider-using-f-string {[common]pyfiles} [testenv:shellcheck] # shellcheck from Xenial is too old and doesn't support the --severity From 0b9a8d230ab2b95d77a00d869c8a675c7d6cc9d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Balatoni?= Date: Wed, 31 Dec 2025 01:11:46 +0100 Subject: [PATCH 05/10] fix(tox): disable new pylint/shellcheck warnings for compatibility --- tox.ini | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index d38088c..cc9d685 100644 --- a/tox.ini +++ b/tox.ini @@ -17,14 +17,14 @@ commands = python3 -m flake8 {[common]pyfiles} [testenv:pylint] deps = pylint==3.0.3 -commands = python3 -m pylint --disable=consider-using-f-string {[common]pyfiles} +commands = python3 -m pylint --disable=consider-using-f-string,unspecified-encoding,consider-using-with {[common]pyfiles} [testenv:shellcheck] # shellcheck from Xenial is too old and doesn't support the --severity # argument; let's pull a newer version using shellcheck-py. This also # allows to pin the shellcheck version we check against. deps = shellcheck-py==0.9.0.6 -commands = shellcheck --severity=error {[common]shfiles} +commands = shellcheck --severity=error --exclude=SC2284 {[common]shfiles} [testenv:tip-flake8] deps = flake8 @@ -36,4 +36,4 @@ commands = python3 -m pylint {[common]pyfiles} [testenv:tip-shellcheck] deps = shellcheck-py -commands = shellcheck --severity=error {[common]shfiles} +commands = shellcheck --severity=error --exclude=SC2284 {[common]shfiles} From 10e4d905124e187243dcd5a2c0d942bed7c77fe0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Balatoni?= Date: Wed, 31 Dec 2025 01:13:46 +0100 Subject: [PATCH 06/10] fix(ci): skip growpart-lvm test in BusyBox environment --- .github/workflows/ci.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 39eeb8b..c7bac04 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -27,8 +27,9 @@ jobs: # SKIP growpart-lvm test that does not work on github c-i run: sudo SKIP=growpart-lvm PATH=$PWD/bin:$PATH ./test/run-all - name: Run tests against BusyBox-based minimal environment + # SKIP growpart-lvm test as LVM tools are not available in BusyBox run: | mkdir /tmp/busybox-bin busybox --install -s /tmp/busybox-bin ln -s $(command -v flock mkfs.ext4 qemu-img qemu-nbd resize2fs sfdisk sgdisk) /tmp/busybox-bin - sudo PATH=$PWD/bin:/tmp/busybox-bin ./test/run-all + sudo SKIP=growpart-lvm PATH=$PWD/bin:/tmp/busybox-bin ./test/run-all From e103ea928e7e8dd0e4edce0fb43085c394af2e01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Balatoni?= Date: Wed, 31 Dec 2025 01:21:39 +0100 Subject: [PATCH 07/10] chore(tox): update to latest linter versions --- tox.ini | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tox.ini b/tox.ini index cc9d685..7613265 100644 --- a/tox.ini +++ b/tox.ini @@ -12,19 +12,19 @@ pyfiles = bin/ec2metadata bin/write-mime-multipart shfiles = bin/cloud-localds bin/growpart bin/mount-image-callback bin/resize-part-image bin/vcs-run test/test-growpart test/test-growpart-fsimage test/test-growpart-fsimage-middle test/test-growpart-overprovision test/test-growpart-lvm test/test-growpart-start-matches-size test/test-mic test/run-all tools/build-deb tools/make-short-partition tools/make-tarball [testenv:flake8] -deps = flake8==7.0.0 +deps = flake8==7.3.0 commands = python3 -m flake8 {[common]pyfiles} [testenv:pylint] -deps = pylint==3.0.3 -commands = python3 -m pylint --disable=consider-using-f-string,unspecified-encoding,consider-using-with {[common]pyfiles} +deps = pylint==4.0.4 +commands = python3 -m pylint {[common]pyfiles} [testenv:shellcheck] # shellcheck from Xenial is too old and doesn't support the --severity # argument; let's pull a newer version using shellcheck-py. This also # allows to pin the shellcheck version we check against. -deps = shellcheck-py==0.9.0.6 -commands = shellcheck --severity=error --exclude=SC2284 {[common]shfiles} +deps = shellcheck-py==0.11.0.1 +commands = shellcheck --severity=error {[common]shfiles} [testenv:tip-flake8] deps = flake8 @@ -36,4 +36,4 @@ commands = python3 -m pylint {[common]pyfiles} [testenv:tip-shellcheck] deps = shellcheck-py -commands = shellcheck --severity=error --exclude=SC2284 {[common]shfiles} +commands = shellcheck --severity=error {[common]shfiles} From 44610d1d31cf22003e5df21e8cbe2d9bd02b843e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Balatoni?= Date: Wed, 31 Dec 2025 01:21:47 +0100 Subject: [PATCH 08/10] refactor(ec2metadata): convert % formatting to f-strings --- bin/ec2metadata | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/bin/ec2metadata b/bin/ec2metadata index a67f6e6..a62c93b 100755 --- a/bin/ec2metadata +++ b/bin/ec2metadata @@ -36,15 +36,15 @@ except ImportError: instdata_host = "169.254.169.254" instdata_ver = "2009-04-04" -instdata_url = "http://%s/%s" % (instdata_host, instdata_ver) +instdata_url = f"http://{instdata_host}/{instdata_ver}" TOKEN_TTL_SECONDS = 21600 TOKEN_HEADER = "X-aws-ec2-metadata-token" TOKEN_HEADER_TTL = "X-aws-ec2-metadata-token-ttl-seconds" -session_token_url = "http://%s/%s/%s" % (instdata_host, 'latest', 'api/token') +session_token_url = f"http://{instdata_host}/latest/api/token" -__doc__ = """ +__doc__ = f""" Query and display EC2 metadata. If no options are provided, all options will be displayed @@ -87,9 +87,9 @@ Options: --public-keys display the openssh public keys --user-data display the user data (not actually metadata) - -u | --url URL use URL (default: %s) + -u | --url URL use URL (default: {instdata_url}) -""" % instdata_url +""" METAOPTS = ['ami-id', 'ami-launch-index', 'ami-manifest-path', @@ -127,8 +127,7 @@ class EC2Metadata: # pylint: disable=R0903 if s.port is None: port = 80 if not self._test_connectivity(addr, port): - raise Error("could not establish connection to: %s:%s" % - (addr, port)) + raise Error(f"could not establish connection to: {addr}:{port}") self._imdsv2_ensure_token() @staticmethod @@ -154,7 +153,7 @@ class EC2Metadata: # pylint: disable=R0903 self.session_token = resp.read() def _get(self, uri, decode=True): - url = "%s/%s" % (self.burl, uri) + url = f"{self.burl}/{uri}" try: resp = urllib_request.urlopen( urllib_request.Request( @@ -198,7 +197,7 @@ class EC2Metadata: # pylint: disable=R0903 public_keys = [] for keyid in keyids: - uri = 'meta-data/public-keys/%d/openssh-key' % int(keyid) + uri = f'meta-data/public-keys/{int(keyid)}/openssh-key' public_keys.append(self._get(uri).rstrip()) return public_keys @@ -226,7 +225,7 @@ def display(metaopts, burl, prefix=False): value = "unavailable" if prefix: - print("%s: %s" % (metaopt, value)) + print(f"{metaopt}: {value}") elif metaopt == "user-data": # We want to avoid binary blob corruption while printing as string print_binary(value) @@ -239,8 +238,8 @@ def usage(s=None): msg = "" if s: - msg = "Error: %s\n" % s - msg += "Syntax: %s [options]\n" % sys.argv[0] + msg = f"Error: {s}\n" + msg += f"Syntax: {sys.argv[0]} [options]\n" msg += __doc__ sys.stderr.write(msg + "\n") sys.exit(1) From 4c5430432b17e6d494098e539b4d6a9f111108e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Balatoni?= Date: Wed, 31 Dec 2025 01:21:53 +0100 Subject: [PATCH 09/10] refactor(write-mime-multipart): use context managers and encoding --- bin/write-mime-multipart | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/bin/write-mime-multipart b/bin/write-mime-multipart index fea9638..6fab346 100755 --- a/bin/write-mime-multipart +++ b/bin/write-mime-multipart @@ -83,15 +83,13 @@ def main(): maintype, subtype = mtype.split('/', 1) if maintype == 'text': - fp = open(path) - # Note: we should handle calculating the charset - msg = MIMEText(fp.read(), _subtype=subtype) - fp.close() + with open(path, encoding='utf-8') as fp: + # Note: we should handle calculating the charset + msg = MIMEText(fp.read(), _subtype=subtype) else: - fp = open(path, 'rb') - msg = MIMEBase(maintype, subtype) - msg.set_payload(fp.read()) - fp.close() + with open(path, 'rb') as fp: + msg = MIMEBase(maintype, subtype) + msg.set_payload(fp.read()) # Encode the payload using Base64 encoders.encode_base64(msg) @@ -107,18 +105,20 @@ def main(): ofile = sys.stdout.buffer else: ofile = sys.stdout + _write_output(ofile, outer, options) else: - ofile = open(options.output, "wb") + with open(options.output, "wb") as ofile: + _write_output(ofile, outer, options) + +def _write_output(ofile, outer, options): + """Write the MIME message to the output file.""" if options.compress: - gfile = gzip.GzipFile(fileobj=ofile, filename=options.output) - gfile.write(outer.as_string().encode()) - gfile.close() + with gzip.GzipFile(fileobj=ofile, filename=options.output) as gfile: + gfile.write(outer.as_string().encode()) else: ofile.write(outer.as_string().encode()) - ofile.close() - if __name__ == '__main__': main() From b4b93b415f2bd0c0d5397e9d1a8b46dfd0d301c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Balatoni?= Date: Wed, 31 Dec 2025 01:22:01 +0100 Subject: [PATCH 10/10] fix(test): quote echo string to fix shellcheck SC2284 --- test/test-growpart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-growpart b/test/test-growpart index 5e917be..ba3f84e 100755 --- a/test/test-growpart +++ b/test/test-growpart @@ -84,7 +84,7 @@ if [ -z "$out" ]; then exit 1 fi -echo === growpart stderr === +echo "=== growpart stderr ===" cat "$errfile" echo "==== after ===="