From 31949e7fa318660d44bab261af15ae5028061a44 Mon Sep 17 00:00:00 2001 From: David Tucker Date: Mon, 7 Aug 2023 18:00:07 +0000 Subject: [PATCH 1/6] Upgrade black --- src/isilon_hadoop_tools/directories.py | 4 +++- tox.ini | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/isilon_hadoop_tools/directories.py b/src/isilon_hadoop_tools/directories.py index 6790de9..08934ee 100644 --- a/src/isilon_hadoop_tools/directories.py +++ b/src/isilon_hadoop_tools/directories.py @@ -181,7 +181,9 @@ def cdp_directories(identity_suffix=None): HDFSDirectory("/user/yarn/mapreduce", "hdfs", "supergroup", 0o775), HDFSDirectory("/user/yarn/mapreduce/mr-framework", "yarn", "hadoop", 0o775), HDFSDirectory("/user/yarn/services", "hdfs", "supergroup", 0o775), - HDFSDirectory("/user/yarn/services/service-framework", "hdfs", "supergroup", 0o775), + HDFSDirectory( + "/user/yarn/services/service-framework", "hdfs", "supergroup", 0o775 + ), HDFSDirectory("/user/zeppelin", "zeppelin", "zeppelin", 0o775), HDFSDirectory("/warehouse", "hdfs", "supergroup", 0o775), HDFSDirectory("/warehouse/tablespace", "hdfs", "supergroup", 0o775), diff --git a/tox.ini b/tox.ini index c5d1855..cccb665 100644 --- a/tox.ini +++ b/tox.ini @@ -25,7 +25,7 @@ python = [testenv:static] basepython = python3.8 deps = - black ~= 23.1.0 + black ~= 23.7.0 flake8 ~= 6.0.0 pylint ~= 2.16.0 commands = From ad82e0af7775079a15b283610c1ffa5ff090fd73 Mon Sep 17 00:00:00 2001 From: David Tucker Date: Mon, 7 Aug 2023 18:27:23 +0000 Subject: [PATCH 2/6] Upgrade pylint --- src/isilon_hadoop_tools/onefs.py | 9 ++++----- tox.ini | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/isilon_hadoop_tools/onefs.py b/src/isilon_hadoop_tools/onefs.py index 1c86171..a7d4bb8 100644 --- a/src/isilon_hadoop_tools/onefs.py +++ b/src/isilon_hadoop_tools/onefs.py @@ -1054,11 +1054,10 @@ def flush_auth_cache(self, zone=None): response.raise_for_status() except requests.exceptions.HTTPError as exc: raise NonSDKAPIError("The auth cache could not be flushed.") from exc - else: - assert bool( - response.status_code - == requests.codes.no_content, # pylint: disable=no-member - ) + assert bool( + response.status_code + == requests.codes.no_content, # pylint: disable=no-member + ) else: try: self._sdk.AuthApi(self._api_client).create_auth_cache_item( diff --git a/tox.ini b/tox.ini index cccb665..b0c1d52 100644 --- a/tox.ini +++ b/tox.ini @@ -27,7 +27,7 @@ basepython = python3.8 deps = black ~= 23.7.0 flake8 ~= 6.0.0 - pylint ~= 2.16.0 + pylint[spelling] ~= 2.17.0 commands = black --check src setup.py tests flake8 src setup.py tests From 52bea583175ce12e1ac7a2360bb5161228befe30 Mon Sep 17 00:00:00 2001 From: David Tucker Date: Mon, 7 Aug 2023 16:41:04 +0000 Subject: [PATCH 3/6] Rename path to zone_path in create_directories --- src/isilon_hadoop_tools/directories.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/isilon_hadoop_tools/directories.py b/src/isilon_hadoop_tools/directories.py index 08934ee..2e82b0b 100644 --- a/src/isilon_hadoop_tools/directories.py +++ b/src/isilon_hadoop_tools/directories.py @@ -60,26 +60,28 @@ def create_directories( if setup: setup(zone_root, hdfs_root, zone_hdfs) for directory in directories: - path = posixpath.join(zone_hdfs, directory.path.lstrip(posixpath.sep)) - LOGGER.info("mkdir '%s%s'", zone_root, path) + zone_path = posixpath.join(zone_hdfs, directory.path.lstrip(posixpath.sep)) + LOGGER.info("mkdir '%s%s'", zone_root, zone_path) try: - (mkdir or self.onefs.mkdir)(path, directory.mode, zone=self.onefs_zone) + (mkdir or self.onefs.mkdir)( + zone_path, directory.mode, zone=self.onefs_zone + ) except isilon_hadoop_tools.onefs.APIError as exc: if exc.dir_path_already_exists_error(): - LOGGER.warning("%s%s already exists. ", zone_root, path) + LOGGER.warning("%s%s already exists. ", zone_root, zone_path) else: raise - LOGGER.info("chmod '%o' '%s%s'", directory.mode, zone_root, path) - (chmod or self.onefs.chmod)(path, directory.mode, zone=self.onefs_zone) + LOGGER.info("chmod '%o' '%s%s'", directory.mode, zone_root, zone_path) + (chmod or self.onefs.chmod)(zone_path, directory.mode, zone=self.onefs_zone) LOGGER.info( "chown '%s:%s' '%s%s'", directory.owner, directory.group, zone_root, - path, + zone_path, ) (chown or self.onefs.chown)( - path, + zone_path, owner=directory.owner, group=directory.group, zone=self.onefs_zone, From 1cd830499ec85290d8babe3159df40b916cd16ac Mon Sep 17 00:00:00 2001 From: David Tucker Date: Mon, 7 Aug 2023 16:57:36 +0000 Subject: [PATCH 4/6] Use sep uniformly in create_directories --- src/isilon_hadoop_tools/directories.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/isilon_hadoop_tools/directories.py b/src/isilon_hadoop_tools/directories.py index 2e82b0b..7fcfb22 100644 --- a/src/isilon_hadoop_tools/directories.py +++ b/src/isilon_hadoop_tools/directories.py @@ -60,7 +60,7 @@ def create_directories( if setup: setup(zone_root, hdfs_root, zone_hdfs) for directory in directories: - zone_path = posixpath.join(zone_hdfs, directory.path.lstrip(posixpath.sep)) + zone_path = posixpath.join(zone_hdfs, directory.path.lstrip(sep)) LOGGER.info("mkdir '%s%s'", zone_root, zone_path) try: (mkdir or self.onefs.mkdir)( From 23a29394a99f2c2fad47b851d5e3dfcb08eb72c3 Mon Sep 17 00:00:00 2001 From: David Tucker Date: Mon, 7 Aug 2023 17:00:03 +0000 Subject: [PATCH 5/6] Use relpath for zone_hdfs in create_directories --- src/isilon_hadoop_tools/directories.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/isilon_hadoop_tools/directories.py b/src/isilon_hadoop_tools/directories.py index 7fcfb22..14bd292 100644 --- a/src/isilon_hadoop_tools/directories.py +++ b/src/isilon_hadoop_tools/directories.py @@ -56,7 +56,7 @@ def create_directories( # and modifying /ifs can break NFS/SMB. raise HDFSRootDirectoryError(hdfs_root) assert hdfs_root.startswith(zone_root) - zone_hdfs = hdfs_root[len(zone_root) :] + zone_hdfs = posixpath.relpath(hdfs_root, start=zone_root) if setup: setup(zone_root, hdfs_root, zone_hdfs) for directory in directories: From 3e25a495e8ecd0cd948582eaa5606f0e5d3709c5 Mon Sep 17 00:00:00 2001 From: David Tucker Date: Fri, 1 Oct 2021 14:15:06 -0700 Subject: [PATCH 6/6] Fix path-handling in create_directories logging --- src/isilon_hadoop_tools/directories.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/isilon_hadoop_tools/directories.py b/src/isilon_hadoop_tools/directories.py index 14bd292..b3bfd44 100644 --- a/src/isilon_hadoop_tools/directories.py +++ b/src/isilon_hadoop_tools/directories.py @@ -61,25 +61,20 @@ def create_directories( setup(zone_root, hdfs_root, zone_hdfs) for directory in directories: zone_path = posixpath.join(zone_hdfs, directory.path.lstrip(sep)) - LOGGER.info("mkdir '%s%s'", zone_root, zone_path) + path = posixpath.join(zone_root, zone_path) + LOGGER.info("mkdir '%s'", path) try: (mkdir or self.onefs.mkdir)( zone_path, directory.mode, zone=self.onefs_zone ) except isilon_hadoop_tools.onefs.APIError as exc: if exc.dir_path_already_exists_error(): - LOGGER.warning("%s%s already exists. ", zone_root, zone_path) + LOGGER.warning("%s already exists. ", path) else: raise - LOGGER.info("chmod '%o' '%s%s'", directory.mode, zone_root, zone_path) + LOGGER.info("chmod '%o' '%s'", directory.mode, path) (chmod or self.onefs.chmod)(zone_path, directory.mode, zone=self.onefs_zone) - LOGGER.info( - "chown '%s:%s' '%s%s'", - directory.owner, - directory.group, - zone_root, - zone_path, - ) + LOGGER.info("chown '%s:%s' '%s'", directory.owner, directory.group, path) (chown or self.onefs.chown)( zone_path, owner=directory.owner,