From 36a8606775b700bfc36ce9e85a23056fc84094ae Mon Sep 17 00:00:00 2001 From: John Parent Date: Tue, 15 Dec 2020 11:57:03 -0500 Subject: [PATCH 01/13] Windows pywin32 locking Locking ranges is not currently supported on Windows. pywintypes.OVERLAPPED in the Windows lock implementation may not be a class member (#22244) Windows: Lock entire file even if lock-by-range is requested (#24183) Windows lock timeout (#25189) Fixup improper cross-platform lock catching Add locking test to Windows CI (#25233) Co-authored-by: lou.lawrence@kitware.com --- lib/spack/spack/llnl/util/lock.py | 130 ++++++++++++++++++++++++++---- 1 file changed, 116 insertions(+), 14 deletions(-) diff --git a/lib/spack/spack/llnl/util/lock.py b/lib/spack/spack/llnl/util/lock.py index 63abafa6ece8aa..77587b636630cb 100644 --- a/lib/spack/spack/llnl/util/lock.py +++ b/lib/spack/spack/llnl/util/lock.py @@ -10,14 +10,21 @@ from datetime import datetime from types import TracebackType from typing import IO, Any, Callable, ContextManager, Dict, Generator, Optional, Tuple, Type, Union +from sys import platform as _platform +from typing import Dict, Tuple # novm from spack.llnl.util import lang, tty from ..string import plural -if sys.platform != "win32": +is_windows = _platform == 'win32' +if not is_windows: import fcntl +else: + import win32con + import win32file + import pywintypes # isort:skip __all__ = [ "Lock", @@ -195,6 +202,26 @@ def is_valid(op: int) -> bool: return op == LockType.READ or op == LockType.WRITE +def lock_checking(func): + from functools import wraps + + @wraps(func) + def win_lock(self, *args, **kwargs): + if is_windows and self._reads > 0: + self._partial_unlock() + try: + suc = func(self, *args, **kwargs) + except Exception as e: + if self._current_lock: + timeout = kwargs.get('timeout', None) + self._lock(self._current_lock, timeout=timeout) + raise e + else: + suc = func(self, *args, **kwargs) + return suc + return win_lock + + class Lock: """This is an implementation of a filesystem lock using Python's lockf. @@ -243,6 +270,7 @@ def __init__( """ self.path = path self._file: Optional[IO[bytes]] = None + self._file_mode = "" self._reads = 0 self._writes = 0 @@ -267,6 +295,32 @@ def __init__( self.host: Optional[str] = None self.old_host: Optional[str] = None + if is_windows: + self.LOCK_EX = win32con.LOCKFILE_EXCLUSIVE_LOCK # exclusive lock + self.LOCK_SH = 0 # shared lock, default + self.LOCK_NB = win32con.LOCKFILE_FAIL_IMMEDIATELY # non-blocking + self.LOCK_CATCH = pywintypes.error + else: + self.LOCK_EX = fcntl.LOCK_EX + self.LOCK_SH = fcntl.LOCK_SH + self.LOCK_NB = fcntl.LOCK_NB + self.LOCK_UN = fcntl.LOCK_UN + self.LOCK_CATCH = IOError + + # Mapping of supported locks to description + self.lock_type = {self.LOCK_SH: 'read', self.LOCK_EX: 'write'} + self._current_lock = None + + def __lock_fail_condition(self, e): + if is_windows: + # 33 "The process cannot access the file because another + # process has locked a portion of the file." + # 32 "The process cannot access the file because it is being + # used by another process" + return e.args[0] not in (32, 33) + else: + return e.errno not in (errno.EAGAIN, errno.EACCES) + @staticmethod def _poll_interval_generator( _wait_times: Optional[Tuple[float, float, float]] = None, @@ -298,6 +352,8 @@ def __repr__(self) -> str: rep = "{0}(".format(self.__class__.__name__) for attr, value in self.__dict__.items(): rep += "{0}={1}, ".format(attr, value.__repr__()) + if attr != "LOCK_CATCH": + rep += '{0}={1}, '.format(attr, value.__repr__()) return "{0})".format(rep.strip(", ")) def __str__(self) -> str: @@ -307,6 +363,7 @@ def __str__(self) -> str: activity = "#reads={0}, #writes={1}".format(self._reads, self._writes) return "({0}, {1}, {2})".format(location, timeout, activity) + @lock_checking def _lock(self, op: int, timeout: Optional[float] = None) -> Tuple[float, int]: """This takes a lock using POSIX locks (``fcntl.lockf``). @@ -362,15 +419,24 @@ def _poll_lock(self, op: int) -> bool: """ assert self._file is not None, "cannot poll a lock without the file being set" module_op = LockType.to_module(op) + try: - # Try to get the lock (will raise if not available.) - fcntl.lockf( - self._file.fileno(), - module_op | fcntl.LOCK_NB, - self._length, - self._start, - os.SEEK_SET, - ) + if is_windows: + hfile = win32file._get_osfhandle(self._file.fileno()) + win32file.LockFileEx(hfile, + module_op | self.LOCK_NB, # flags + 0, + 0xffff0000, + pywintypes.OVERLAPPED()) + else: + # Try to get the lock (will raise if not available.) + fcntl.lockf( + self._file.fileno(), + module_op | fcntl.LOCK_NB, + self._length, + self._start, + os.SEEK_SET, + ) # help for debugging distributed locking if self.debug: @@ -383,7 +449,7 @@ def _poll_lock(self, op: int) -> bool: ) # Exclusive locks write their PID/host - if module_op == fcntl.LOCK_EX: + if op == self.LOCK_EX: self._write_log_debug_data() return True @@ -392,12 +458,15 @@ def _poll_lock(self, op: int) -> bool: # EAGAIN and EACCES == locked by another process (so try again) if e.errno not in (errno.EAGAIN, errno.EACCES): raise + except self.LOCK_CATCH as e: + # check if lock failure or lock is already held + if self.__lock_fail_condition(e): + raise return False def _ensure_parent_directory(self) -> str: parent = os.path.dirname(self.path) - # relative paths to lockfiles in the current directory have no parent if not parent: return "." @@ -407,6 +476,10 @@ def _ensure_parent_directory(self) -> str: def _read_log_debug_data(self) -> None: """Read PID and host data out of the file if it is there.""" assert self._file is not None, "cannot read debug log without the file being set" + if is_windows: + # Not implemented for windows + return + self.old_pid = self.pid self.old_host = self.host @@ -420,6 +493,10 @@ def _read_log_debug_data(self) -> None: def _write_log_debug_data(self) -> None: """Write PID and host data to the file, recording old values.""" assert self._file is not None, "cannot write debug log without the file being set" + if is_windows: + # Not implemented for windows + return + self.old_pid = self.pid self.old_host = self.host @@ -433,7 +510,7 @@ def _write_log_debug_data(self) -> None: self._file.flush() os.fsync(self._file.fileno()) - def _unlock(self) -> None: + def _partial_unlock(self) -> None: """Releases a lock using POSIX locks (``fcntl.lockf``) Releases the lock regardless of mode. Note that read locks may @@ -443,7 +520,30 @@ def _unlock(self) -> None: assert self._file is not None, "cannot unlock without the file being set" fcntl.lockf(self._file.fileno(), fcntl.LOCK_UN, self._length, self._start, os.SEEK_SET) FILE_TRACKER.release_by_fh(self._file) + + if is_windows: + hfile = win32file._get_osfhandle(self._file.fileno()) + win32file.UnlockFileEx(hfile, + 0, + 0xffff0000, + pywintypes.OVERLAPPED()) + else: + fcntl.lockf(self._file, self.LOCK_UN, + self._length, self._start, os.SEEK_SET) + + def _unlock(self): + """Releases a lock using POSIX locks (``fcntl.lockf``) + + Releases the lock regardless of mode. Note that read locks may + be masquerading as write locks, but this removes either. + + Reset all lock attributes to initial states + """ + self._partial_unlock() + file_tracker.release_fh(self.path) + self._file = None + self._file_mode = "" self._reads = 0 self._writes = 0 @@ -464,6 +564,7 @@ def acquire_read(self, timeout: Optional[float] = None) -> bool: # can raise LockError. wait_time, nattempts = self._lock(LockType.READ, timeout=timeout) self._reads += 1 + self._current_lock = self.LOCK_SH # Log if acquired, which includes counts when verbose self._log_acquired("READ LOCK", wait_time, nattempts) return True @@ -489,6 +590,7 @@ def acquire_write(self, timeout: Optional[float] = None) -> bool: # can raise LockError. wait_time, nattempts = self._lock(LockType.WRITE, timeout=timeout) self._writes += 1 + self._current_lock = self.LOCK_EX # Log if acquired, which includes counts when verbose self._log_acquired("WRITE LOCK", wait_time, nattempts) @@ -528,7 +630,7 @@ def downgrade_write_to_read(self, timeout: Optional[float] = None) -> None: """ timeout = timeout or self.default_timeout - if self._writes == 1 and self._reads == 0: + if self._writes == 1: self._log_downgrading() # can raise LockError. wait_time, nattempts = self._lock(LockType.READ, timeout=timeout) @@ -547,7 +649,7 @@ def upgrade_read_to_write(self, timeout: Optional[float] = None) -> None: """ timeout = timeout or self.default_timeout - if self._reads == 1 and self._writes == 0: + if self._reads >= 1 and self._writes == 0: self._log_upgrading() # can raise LockError. wait_time, nattempts = self._lock(LockType.WRITE, timeout=timeout) From 025508571d7c1d654a780aaaed539f0506a342be Mon Sep 17 00:00:00 2001 From: John Parent Date: Tue, 25 Jan 2022 17:30:20 -0500 Subject: [PATCH 02/13] Add locking Windows actions --- share/spack/qa/configuration/windows_locking_config.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 share/spack/qa/configuration/windows_locking_config.yaml diff --git a/share/spack/qa/configuration/windows_locking_config.yaml b/share/spack/qa/configuration/windows_locking_config.yaml new file mode 100644 index 00000000000000..266c7c42df93b0 --- /dev/null +++ b/share/spack/qa/configuration/windows_locking_config.yaml @@ -0,0 +1,8 @@ +config: + locks: true + install_tree: + root: $spack\opt\spack + projections: + all: '${ARCHITECTURE}\${COMPILERNAME}-${COMPILERVER}\${PACKAGE}-${VERSION}-${HASH}' + build_stage: + - ~/.spack/stage From 866008823a85dcc10a99bd4a6e4278d2df48c639 Mon Sep 17 00:00:00 2001 From: John Parent Date: Thu, 7 Apr 2022 14:16:21 -0400 Subject: [PATCH 03/13] Update locks for win --- etc/spack/defaults/windows/config.yaml | 2 +- lib/spack/spack/llnl/util/lock.py | 29 +++++++++++++------------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/etc/spack/defaults/windows/config.yaml b/etc/spack/defaults/windows/config.yaml index f54febe957553e..af90cc33b75c8d 100644 --- a/etc/spack/defaults/windows/config.yaml +++ b/etc/spack/defaults/windows/config.yaml @@ -1,5 +1,5 @@ config: - locks: false + locks: true build_stage:: - '$user_cache_path/stage' stage_name: '{name}-{version}-{hash:7}' diff --git a/lib/spack/spack/llnl/util/lock.py b/lib/spack/spack/llnl/util/lock.py index 77587b636630cb..a47cc213196600 100644 --- a/lib/spack/spack/llnl/util/lock.py +++ b/lib/spack/spack/llnl/util/lock.py @@ -182,6 +182,17 @@ def _attempts_str(wait_time, nattempts): class LockType: READ = 0 WRITE = 1 + if is_windows: + LOCK_EX = win32con.LOCKFILE_EXCLUSIVE_LOCK # exclusive lock + LOCK_SH = 0 # shared lock, default + LOCK_NB = win32con.LOCKFILE_FAIL_IMMEDIATELY # non-blocking + LOCK_CATCH = pywintypes.error + else: + LOCK_EX = fcntl.LOCK_EX + LOCK_SH = fcntl.LOCK_SH + LOCK_NB = fcntl.LOCK_NB + LOCK_UN = fcntl.LOCK_UN + LOCK_CATCH = IOError @staticmethod def to_str(tid): @@ -192,9 +203,9 @@ def to_str(tid): @staticmethod def to_module(tid): - lock = fcntl.LOCK_SH + lock = LockType.LOCK_SH if tid == LockType.WRITE: - lock = fcntl.LOCK_EX + lock = LockType.LOCK_EX return lock @staticmethod @@ -295,18 +306,6 @@ def __init__( self.host: Optional[str] = None self.old_host: Optional[str] = None - if is_windows: - self.LOCK_EX = win32con.LOCKFILE_EXCLUSIVE_LOCK # exclusive lock - self.LOCK_SH = 0 # shared lock, default - self.LOCK_NB = win32con.LOCKFILE_FAIL_IMMEDIATELY # non-blocking - self.LOCK_CATCH = pywintypes.error - else: - self.LOCK_EX = fcntl.LOCK_EX - self.LOCK_SH = fcntl.LOCK_SH - self.LOCK_NB = fcntl.LOCK_NB - self.LOCK_UN = fcntl.LOCK_UN - self.LOCK_CATCH = IOError - # Mapping of supported locks to description self.lock_type = {self.LOCK_SH: 'read', self.LOCK_EX: 'write'} self._current_lock = None @@ -385,7 +384,7 @@ def _lock(self, op: int, timeout: Optional[float] = None) -> Tuple[float, int]: self._ensure_parent_directory() self._file = FILE_TRACKER.get_fh(self.path) - if LockType.to_module(op) == fcntl.LOCK_EX and self._file.mode == "rb": + if LockType.to_module(op) == LockType.LOCK_EX and self._file.mode == "rb": # Attempt to upgrade to write lock w/a read-only file. # If the file were writable, we'd have opened it rb+ raise LockROFileError(self.path) From 59d838b3f027b730708d817dd933c02ce4d1435c Mon Sep 17 00:00:00 2001 From: John Parent Date: Wed, 9 Jul 2025 19:09:42 -0400 Subject: [PATCH 04/13] Remove locking guard on Windows --- lib/spack/spack/util/lock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/util/lock.py b/lib/spack/spack/util/lock.py index e407f7099cbd67..b6210d2d095ae0 100644 --- a/lib/spack/spack/util/lock.py +++ b/lib/spack/spack/util/lock.py @@ -38,7 +38,7 @@ def __init__( desc: str = "", enable: bool = True, ) -> None: - self._enable = sys.platform != "win32" and enable + self._enable = enable super().__init__( path, start=start, From a38a30ab8f2f1f00f1a5c59d5ea3c4d93935990c Mon Sep 17 00:00:00 2001 From: John Parent Date: Fri, 11 Jul 2025 10:25:35 -0400 Subject: [PATCH 05/13] Style --- lib/spack/spack/llnl/util/lock.py | 61 ++++++++++++++++++------------- 1 file changed, 35 insertions(+), 26 deletions(-) diff --git a/lib/spack/spack/llnl/util/lock.py b/lib/spack/spack/llnl/util/lock.py index a47cc213196600..4adb1cf0dd5a68 100644 --- a/lib/spack/spack/llnl/util/lock.py +++ b/lib/spack/spack/llnl/util/lock.py @@ -8,17 +8,27 @@ import sys import time from datetime import datetime -from types import TracebackType -from typing import IO, Any, Callable, ContextManager, Dict, Generator, Optional, Tuple, Type, Union from sys import platform as _platform -from typing import Dict, Tuple # novm +from types import TracebackType +from typing import ( # novm + IO, + Any, + Callable, + ContextManager, + Dict, + Generator, + Optional, + Tuple, + Type, + Union, +) from spack.llnl.util import lang, tty from ..string import plural -is_windows = _platform == 'win32' -if not is_windows: +IS_WINDOWS = _platform == "win32" +if not IS_WINDOWS: import fcntl else: import win32con @@ -182,7 +192,7 @@ def _attempts_str(wait_time, nattempts): class LockType: READ = 0 WRITE = 1 - if is_windows: + if IS_WINDOWS: LOCK_EX = win32con.LOCKFILE_EXCLUSIVE_LOCK # exclusive lock LOCK_SH = 0 # shared lock, default LOCK_NB = win32con.LOCKFILE_FAIL_IMMEDIATELY # non-blocking @@ -218,18 +228,19 @@ def lock_checking(func): @wraps(func) def win_lock(self, *args, **kwargs): - if is_windows and self._reads > 0: + if IS_WINDOWS and self._reads > 0: self._partial_unlock() try: suc = func(self, *args, **kwargs) except Exception as e: if self._current_lock: - timeout = kwargs.get('timeout', None) + timeout = kwargs.get("timeout", None) self._lock(self._current_lock, timeout=timeout) raise e else: suc = func(self, *args, **kwargs) return suc + return win_lock @@ -307,11 +318,11 @@ def __init__( self.old_host: Optional[str] = None # Mapping of supported locks to description - self.lock_type = {self.LOCK_SH: 'read', self.LOCK_EX: 'write'} + self.lock_type = {self.LOCK_SH: "read", self.LOCK_EX: "write"} self._current_lock = None def __lock_fail_condition(self, e): - if is_windows: + if IS_WINDOWS: # 33 "The process cannot access the file because another # process has locked a portion of the file." # 32 "The process cannot access the file because it is being @@ -352,7 +363,7 @@ def __repr__(self) -> str: for attr, value in self.__dict__.items(): rep += "{0}={1}, ".format(attr, value.__repr__()) if attr != "LOCK_CATCH": - rep += '{0}={1}, '.format(attr, value.__repr__()) + rep += "{0}={1}, ".format(attr, value.__repr__()) return "{0})".format(rep.strip(", ")) def __str__(self) -> str: @@ -420,13 +431,15 @@ def _poll_lock(self, op: int) -> bool: module_op = LockType.to_module(op) try: - if is_windows: + if IS_WINDOWS: hfile = win32file._get_osfhandle(self._file.fileno()) - win32file.LockFileEx(hfile, - module_op | self.LOCK_NB, # flags - 0, - 0xffff0000, - pywintypes.OVERLAPPED()) + win32file.LockFileEx( + hfile, + module_op | self.LOCK_NB, # flags + 0, + 0xFFFF0000, + pywintypes.OVERLAPPED(), + ) else: # Try to get the lock (will raise if not available.) fcntl.lockf( @@ -475,7 +488,7 @@ def _ensure_parent_directory(self) -> str: def _read_log_debug_data(self) -> None: """Read PID and host data out of the file if it is there.""" assert self._file is not None, "cannot read debug log without the file being set" - if is_windows: + if IS_WINDOWS: # Not implemented for windows return @@ -492,7 +505,7 @@ def _read_log_debug_data(self) -> None: def _write_log_debug_data(self) -> None: """Write PID and host data to the file, recording old values.""" assert self._file is not None, "cannot write debug log without the file being set" - if is_windows: + if IS_WINDOWS: # Not implemented for windows return @@ -520,15 +533,11 @@ def _partial_unlock(self) -> None: fcntl.lockf(self._file.fileno(), fcntl.LOCK_UN, self._length, self._start, os.SEEK_SET) FILE_TRACKER.release_by_fh(self._file) - if is_windows: + if IS_WINDOWS: hfile = win32file._get_osfhandle(self._file.fileno()) - win32file.UnlockFileEx(hfile, - 0, - 0xffff0000, - pywintypes.OVERLAPPED()) + win32file.UnlockFileEx(hfile, 0, 0xFFFF0000, pywintypes.OVERLAPPED()) else: - fcntl.lockf(self._file, self.LOCK_UN, - self._length, self._start, os.SEEK_SET) + fcntl.lockf(self._file, self.LOCK_UN, self._length, self._start, os.SEEK_SET) def _unlock(self): """Releases a lock using POSIX locks (``fcntl.lockf``) From 5817654eddcddfddb35d53de742051687641f86f Mon Sep 17 00:00:00 2001 From: John Parent Date: Fri, 11 Jul 2025 11:33:14 -0400 Subject: [PATCH 06/13] Fixup import --- lib/spack/spack/llnl/util/lock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/llnl/util/lock.py b/lib/spack/spack/llnl/util/lock.py index 4adb1cf0dd5a68..e99da6ac8bdb00 100644 --- a/lib/spack/spack/llnl/util/lock.py +++ b/lib/spack/spack/llnl/util/lock.py @@ -31,10 +31,10 @@ if not IS_WINDOWS: import fcntl else: + import pywintypes import win32con import win32file - import pywintypes # isort:skip __all__ = [ "Lock", From b5fc65f5c0af424e84b18d27298a85a467167c8c Mon Sep 17 00:00:00 2001 From: John Parent Date: Fri, 11 Jul 2025 14:43:27 -0400 Subject: [PATCH 07/13] wip --- lib/spack/spack/llnl/util/lock.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/llnl/util/lock.py b/lib/spack/spack/llnl/util/lock.py index e99da6ac8bdb00..46eb06e254e1bd 100644 --- a/lib/spack/spack/llnl/util/lock.py +++ b/lib/spack/spack/llnl/util/lock.py @@ -530,14 +530,13 @@ def _partial_unlock(self) -> None: """ assert self._file is not None, "cannot unlock without the file being set" - fcntl.lockf(self._file.fileno(), fcntl.LOCK_UN, self._length, self._start, os.SEEK_SET) - FILE_TRACKER.release_by_fh(self._file) if IS_WINDOWS: hfile = win32file._get_osfhandle(self._file.fileno()) win32file.UnlockFileEx(hfile, 0, 0xFFFF0000, pywintypes.OVERLAPPED()) else: - fcntl.lockf(self._file, self.LOCK_UN, self._length, self._start, os.SEEK_SET) + fcntl.lockf(self._file.fileno(), fcntl.LOCK_UN, self._length, self._start, os.SEEK_SET) + def _unlock(self): """Releases a lock using POSIX locks (``fcntl.lockf``) @@ -548,7 +547,7 @@ def _unlock(self): Reset all lock attributes to initial states """ self._partial_unlock() - file_tracker.release_fh(self.path) + FILE_TRACKER.release_by_fh(self._file) self._file = None self._file_mode = "" From 841e98847dc9270708077a833b7a79e98daabf9c Mon Sep 17 00:00:00 2001 From: John Parent Date: Tue, 2 Dec 2025 18:55:00 -0500 Subject: [PATCH 08/13] implement partial locking Signed-off-by: John Parent --- lib/spack/spack/llnl/util/lock.py | 44 +++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/lib/spack/spack/llnl/util/lock.py b/lib/spack/spack/llnl/util/lock.py index 46eb06e254e1bd..07d7cac6624744 100644 --- a/lib/spack/spack/llnl/util/lock.py +++ b/lib/spack/spack/llnl/util/lock.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +import contextlib import errno import os import socket @@ -189,6 +190,20 @@ def _attempts_str(wait_time, nattempts): return " after {} and {}".format(lang.pretty_seconds(wait_time), attempts) +@contextlib.contextmanager +def _partial_upgrade(path:str, start:int = 0, length:int = 0, timeout:Optional[int] = None): + lock_path = path + ".lock" + l = Lock(lock_path, start=start, length=length) + try: + l.acquire_write(timeout=timeout) + yield + except LockError as e: + pass + finally: + if l.is_write_locked(): + l.release_write() + + class LockType: READ = 0 WRITE = 1 @@ -317,8 +332,6 @@ def __init__( self.host: Optional[str] = None self.old_host: Optional[str] = None - # Mapping of supported locks to description - self.lock_type = {self.LOCK_SH: "read", self.LOCK_EX: "write"} self._current_lock = None def __lock_fail_condition(self, e): @@ -373,6 +386,8 @@ def __str__(self) -> str: activity = "#reads={0}, #writes={1}".format(self._reads, self._writes) return "({0}, {1}, {2})".format(location, timeout, activity) + + @lock_checking def _lock(self, op: int, timeout: Optional[float] = None) -> Tuple[float, int]: """This takes a lock using POSIX locks (``fcntl.lockf``). @@ -435,7 +450,7 @@ def _poll_lock(self, op: int) -> bool: hfile = win32file._get_osfhandle(self._file.fileno()) win32file.LockFileEx( hfile, - module_op | self.LOCK_NB, # flags + module_op | LockType.LOCK_NB, # flags 0, 0xFFFF0000, pywintypes.OVERLAPPED(), @@ -444,7 +459,7 @@ def _poll_lock(self, op: int) -> bool: # Try to get the lock (will raise if not available.) fcntl.lockf( self._file.fileno(), - module_op | fcntl.LOCK_NB, + module_op | LockType.LOCK_NB, self._length, self._start, os.SEEK_SET, @@ -461,7 +476,7 @@ def _poll_lock(self, op: int) -> bool: ) # Exclusive locks write their PID/host - if op == self.LOCK_EX: + if op == LockType.LOCK_EX: self._write_log_debug_data() return True @@ -470,7 +485,7 @@ def _poll_lock(self, op: int) -> bool: # EAGAIN and EACCES == locked by another process (so try again) if e.errno not in (errno.EAGAIN, errno.EACCES): raise - except self.LOCK_CATCH as e: + except LockType.LOCK_CATCH as e: # check if lock failure or lock is already held if self.__lock_fail_condition(e): raise @@ -535,7 +550,7 @@ def _partial_unlock(self) -> None: hfile = win32file._get_osfhandle(self._file.fileno()) win32file.UnlockFileEx(hfile, 0, 0xFFFF0000, pywintypes.OVERLAPPED()) else: - fcntl.lockf(self._file.fileno(), fcntl.LOCK_UN, self._length, self._start, os.SEEK_SET) + fcntl.lockf(self._file.fileno(), LockType.LOCK_UN, self._length, self._start, os.SEEK_SET) def _unlock(self): @@ -571,7 +586,7 @@ def acquire_read(self, timeout: Optional[float] = None) -> bool: # can raise LockError. wait_time, nattempts = self._lock(LockType.READ, timeout=timeout) self._reads += 1 - self._current_lock = self.LOCK_SH + self._current_lock = LockType.LOCK_SH # Log if acquired, which includes counts when verbose self._log_acquired("READ LOCK", wait_time, nattempts) return True @@ -597,7 +612,7 @@ def acquire_write(self, timeout: Optional[float] = None) -> bool: # can raise LockError. wait_time, nattempts = self._lock(LockType.WRITE, timeout=timeout) self._writes += 1 - self._current_lock = self.LOCK_EX + self._current_lock = LockType.LOCK_EX # Log if acquired, which includes counts when verbose self._log_acquired("WRITE LOCK", wait_time, nattempts) @@ -658,11 +673,12 @@ def upgrade_read_to_write(self, timeout: Optional[float] = None) -> None: if self._reads >= 1 and self._writes == 0: self._log_upgrading() - # can raise LockError. - wait_time, nattempts = self._lock(LockType.WRITE, timeout=timeout) - self._reads = 0 - self._writes = 1 - self._log_upgraded(wait_time, nattempts) + with partial_upgrade(self.path): + # can raise LockError. + wait_time, nattempts = self._lock(LockType.WRITE, timeout=timeout) + self._reads = 0 + self._writes = 1 + self._log_upgraded(wait_time, nattempts) else: raise LockUpgradeError(self.path) From 7d39cce08b1d7630ad5177c5561b99cbfece2fc3 Mon Sep 17 00:00:00 2001 From: John Parent Date: Thu, 4 Dec 2025 18:59:11 -0500 Subject: [PATCH 09/13] Switchover locking Signed-off-by: John Parent --- lib/spack/spack/llnl/util/lock.py | 4 ++-- lib/spack/spack/test/llnl/util/lock.py | 2 +- lib/spack/spack/util/lock.py | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/llnl/util/lock.py b/lib/spack/spack/llnl/util/lock.py index 07d7cac6624744..f0d4ebb1b84314 100644 --- a/lib/spack/spack/llnl/util/lock.py +++ b/lib/spack/spack/llnl/util/lock.py @@ -6,7 +6,6 @@ import errno import os import socket -import sys import time from datetime import datetime from sys import platform as _platform @@ -192,6 +191,7 @@ def _attempts_str(wait_time, nattempts): @contextlib.contextmanager def _partial_upgrade(path:str, start:int = 0, length:int = 0, timeout:Optional[int] = None): + # probably something like foo.lock.lock lock_path = path + ".lock" l = Lock(lock_path, start=start, length=length) try: @@ -673,7 +673,7 @@ def upgrade_read_to_write(self, timeout: Optional[float] = None) -> None: if self._reads >= 1 and self._writes == 0: self._log_upgrading() - with partial_upgrade(self.path): + with _partial_upgrade(self.path): # can raise LockError. wait_time, nattempts = self._lock(LockType.WRITE, timeout=timeout) self._reads = 0 diff --git a/lib/spack/spack/test/llnl/util/lock.py b/lib/spack/spack/test/llnl/util/lock.py index 26262132100e26..5934b5325c6efe 100644 --- a/lib/spack/spack/test/llnl/util/lock.py +++ b/lib/spack/spack/test/llnl/util/lock.py @@ -64,7 +64,7 @@ if sys.platform != "win32": import fcntl -pytestmark = pytest.mark.not_on_windows("does not run on windows") +# pytestmark = pytest.mark.not_on_windows("does not run on windows") # diff --git a/lib/spack/spack/util/lock.py b/lib/spack/spack/util/lock.py index b6210d2d095ae0..0f085a3c05759b 100644 --- a/lib/spack/spack/util/lock.py +++ b/lib/spack/spack/util/lock.py @@ -5,7 +5,6 @@ """Wrapper for ``spack.llnl.util.lock`` allows locking to be enabled/disabled.""" import os import stat -import sys from typing import Optional, Tuple import spack.error From 415322f5165660c7e2c6933ab86ee843d5beda74 Mon Sep 17 00:00:00 2001 From: John Parent Date: Mon, 8 Dec 2025 17:14:41 -0500 Subject: [PATCH 10/13] Remove old lock upgrade approach Signed-off-by: John Parent --- lib/spack/spack/llnl/util/lock.py | 37 ++----------------------------- 1 file changed, 2 insertions(+), 35 deletions(-) diff --git a/lib/spack/spack/llnl/util/lock.py b/lib/spack/spack/llnl/util/lock.py index f0d4ebb1b84314..930335308613c5 100644 --- a/lib/spack/spack/llnl/util/lock.py +++ b/lib/spack/spack/llnl/util/lock.py @@ -238,27 +238,6 @@ def is_valid(op: int) -> bool: return op == LockType.READ or op == LockType.WRITE -def lock_checking(func): - from functools import wraps - - @wraps(func) - def win_lock(self, *args, **kwargs): - if IS_WINDOWS and self._reads > 0: - self._partial_unlock() - try: - suc = func(self, *args, **kwargs) - except Exception as e: - if self._current_lock: - timeout = kwargs.get("timeout", None) - self._lock(self._current_lock, timeout=timeout) - raise e - else: - suc = func(self, *args, **kwargs) - return suc - - return win_lock - - class Lock: """This is an implementation of a filesystem lock using Python's lockf. @@ -387,8 +366,6 @@ def __str__(self) -> str: return "({0}, {1}, {2})".format(location, timeout, activity) - - @lock_checking def _lock(self, op: int, timeout: Optional[float] = None) -> Tuple[float, int]: """This takes a lock using POSIX locks (``fcntl.lockf``). @@ -537,12 +514,13 @@ def _write_log_debug_data(self) -> None: self._file.flush() os.fsync(self._file.fileno()) - def _partial_unlock(self) -> None: + def _unlock(self): """Releases a lock using POSIX locks (``fcntl.lockf``) Releases the lock regardless of mode. Note that read locks may be masquerading as write locks, but this removes either. + Reset all lock attributes to initial states """ assert self._file is not None, "cannot unlock without the file being set" @@ -551,17 +529,6 @@ def _partial_unlock(self) -> None: win32file.UnlockFileEx(hfile, 0, 0xFFFF0000, pywintypes.OVERLAPPED()) else: fcntl.lockf(self._file.fileno(), LockType.LOCK_UN, self._length, self._start, os.SEEK_SET) - - - def _unlock(self): - """Releases a lock using POSIX locks (``fcntl.lockf``) - - Releases the lock regardless of mode. Note that read locks may - be masquerading as write locks, but this removes either. - - Reset all lock attributes to initial states - """ - self._partial_unlock() FILE_TRACKER.release_by_fh(self._file) self._file = None From 08d3d353ca39d097af3cd9bdeb556321843a83c2 Mon Sep 17 00:00:00 2001 From: John Parent Date: Mon, 8 Dec 2025 18:43:47 -0500 Subject: [PATCH 11/13] win32file locking test Signed-off-by: John Parent --- lib/spack/spack/test/llnl/util/lock.py | 37 +++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/test/llnl/util/lock.py b/lib/spack/spack/test/llnl/util/lock.py index 5934b5325c6efe..06b180b185e797 100644 --- a/lib/spack/spack/test/llnl/util/lock.py +++ b/lib/spack/spack/test/llnl/util/lock.py @@ -63,6 +63,8 @@ if sys.platform != "win32": import fcntl +else: + import win32file # pytestmark = pytest.mark.not_on_windows("does not run on windows") @@ -1324,7 +1326,7 @@ def test_downgrade_write_fails(tmp_path: pathlib.Path): lock.downgrade_write_to_read() lock.release_read() - +@pytest.mark.skipif(sys.platform=="win32", reason="fcntl unavailable on Windows") @pytest.mark.parametrize( "err_num,err_msg", [ @@ -1347,14 +1349,43 @@ def _lockf(fd, cmd, len, start, whence): monkeypatch.setattr(fcntl, "lockf", _lockf) if err_num in [errno.EAGAIN, errno.EACCES]: - assert not lock._poll_lock(fcntl.LOCK_EX) + assert not lock._poll_lock(lk.LockType.LOCK_EX) else: with pytest.raises(OSError, match=err_msg): - lock._poll_lock(fcntl.LOCK_EX) + lock._poll_lock(lk.LockType.LOCK_EX) monkeypatch.undo() lock.release_read() +@pytest.mark.skipif(sys.platform != "win32", reason="win32file only available on Windows") +@pytest.mark.parametrize( + "err_num,err_msg", + [ + (32, "Fake EACCES error analog"), + (33, "Fake EAGAIN error analog"), + ], +) +def test_poll_lock_exception_win(tmp_path: pathlib.Path, monkeypatch, err_num, err_msg): + """Test poll lock exception handling.""" + + def LockFileEx(hfile, int_, int1_, int2_, ol): + raise OSError(err_num, err_msg) + + with working_dir(str(tmp_path)): + lockfile = "lockfile" + lock = lk.Lock(lockfile) + lock.acquire_read() + + monkeypatch.setattr(win32file, "LockFileEx", LockFileEx) + + if err_num in [errno.EAGAIN, errno.EACCES]: + assert not lock._poll_lock(lk.LockType.LOCK_EX) + else: + with pytest.raises(OSError, match=err_msg): + lock._poll_lock(lk.LockType.LOCK_EX) + + monkeypatch.undo() + lock.release_read() def test_upgrade_read_okay(tmp_path: pathlib.Path): """Test the lock read-to-write upgrade operation.""" From 8b930c6baf7bebcc90c8ac7fd1648da09a9e0cf5 Mon Sep 17 00:00:00 2001 From: John Parent Date: Tue, 6 Jan 2026 15:31:36 -0500 Subject: [PATCH 12/13] Support ranges properly Signed-off-by: John Parent --- lib/spack/spack/llnl/util/lock.py | 158 ++++++++++++++++++------- lib/spack/spack/test/llnl/util/lock.py | 15 +-- 2 files changed, 125 insertions(+), 48 deletions(-) diff --git a/lib/spack/spack/llnl/util/lock.py b/lib/spack/spack/llnl/util/lock.py index 930335308613c5..04396651bbb721 100644 --- a/lib/spack/spack/llnl/util/lock.py +++ b/lib/spack/spack/llnl/util/lock.py @@ -50,6 +50,8 @@ "CantCreateLockError", ] +WHOLE_FILE_RANGE = 0XFFFFFFFF if IS_WINDOWS else 0 + ReleaseFnType = Optional[Callable[[], bool]] @@ -189,21 +191,93 @@ def _attempts_str(wait_time, nattempts): return " after {} and {}".format(lang.pretty_seconds(wait_time), attempts) +def _low_high(value): + low = value & 0XFFFFFFFF + high = (value >> 32) & 0XFFFFFFFF + return low, high + + +def _setup_overlapped(offset): + overlapped = pywintypes.OVERLAPPED() + # hEvent needs to be null per lockfileex docs + overlapped.hEvent = 0 + offset_low, offset_high = _low_high(offset) + overlapped.Offset = offset_low + overlapped.OffsetHigh = offset_high + return overlapped + + @contextlib.contextmanager -def _partial_upgrade(path:str, start:int = 0, length:int = 0, timeout:Optional[int] = None): +def _safe_exclusion(lock: "Lock", timeout: Optional[float] = None): + """Lock upgrade guard for Windows, designed to allow for lock upgrading + which Windows file locks do not natively support + Uses one additional lockfile as a "gate" for upgrades. + + If a process is attempting to upgrade from a read to a write + it must first take a write lock on this intermediate file + before releasing existing read lock and retaking a lock on the same + file but exclusively. + This gate file prevents any other process/lock from catching + the lock with a competing write during the release part of the upgrade. + """ # probably something like foo.lock.lock - lock_path = path + ".lock" - l = Lock(lock_path, start=start, length=length) + timeout = timeout or lock.default_timeout + lock_path = lock.path + ".lock" + l = Lock(lock_path, start=lock._start, length=lock._length) + # cache previous value for read locks + _reads = lock._reads try: - l.acquire_write(timeout=timeout) + # don't use the acquire lock method to avoid + # recursion + l._lock(LockType.WRITE, timeout=timeout) + # lock gate acquired, drop read lock if there is one + # cannot use release read as we need to drop + # the read lock if there is one, not just decrement the nested + # lock tracker + if _reads: + lock._release_lock() yield except LockError as e: - pass + # doesn't matter what the error was + # if there was a read lock previously + # restore it + if _reads > 0: + lock._lock(LockType.READ, timeout=timeout) + raise finally: if l.is_write_locked(): l.release_write() +@contextlib.contextmanager +def _safe_downgrade(lock: "Lock", timeout: Optional[float] = None): + """Windows can hold shared and exclusive locks on the same time, but only + if the exclusive lock is held first. When the shared lock overlaps the exclusive + range, the exclusive lock must also be unlocked and must be unlocked *first* + From the MSFT docs: + + A shared lock can overlap an exclusive lock if both locks were created using + the same file handle. If the same range is locked with an exclusive and a + shared lock, two unlock operations are necessary to unlock the region; the + first unlock operation unlocks the exclusive lock, the second unlock + operation unlocks the shared lock. + + Since we are "downgrading" we no longer care to have an exclusive lock + so we just give it up + """ + timeout = timeout or lock.default_timeout + # release the read lock first + # so in the event on an error, we still have the exclusive lock + yield + # From the docstring: + # first unlock operation unlocks the exclusive lock, the second unlock + # operation unlocks the shared lock. + # + # This will remove the exclusive lock + lock._release_lock() + + + class LockType: READ = 0 WRITE = 1 @@ -258,7 +332,7 @@ def __init__( path: str, *, start: int = 0, - length: int = 0, + length: int = WHOLE_FILE_RANGE, default_timeout: Optional[float] = None, debug: bool = False, desc: str = "", @@ -424,13 +498,15 @@ def _poll_lock(self, op: int) -> bool: try: if IS_WINDOWS: + overlapped = _setup_overlapped(self._start) + range_low, range_high = _low_high(self._length) hfile = win32file._get_osfhandle(self._file.fileno()) win32file.LockFileEx( hfile, module_op | LockType.LOCK_NB, # flags - 0, - 0xFFFF0000, - pywintypes.OVERLAPPED(), + range_low, + range_high, + overlapped, ) else: # Try to get the lock (will raise if not available.) @@ -453,7 +529,7 @@ def _poll_lock(self, op: int) -> bool: ) # Exclusive locks write their PID/host - if op == LockType.LOCK_EX: + if op == LockType.WRITE: self._write_log_debug_data() return True @@ -480,9 +556,6 @@ def _ensure_parent_directory(self) -> str: def _read_log_debug_data(self) -> None: """Read PID and host data out of the file if it is there.""" assert self._file is not None, "cannot read debug log without the file being set" - if IS_WINDOWS: - # Not implemented for windows - return self.old_pid = self.pid self.old_host = self.host @@ -497,16 +570,12 @@ def _read_log_debug_data(self) -> None: def _write_log_debug_data(self) -> None: """Write PID and host data to the file, recording old values.""" assert self._file is not None, "cannot write debug log without the file being set" - if IS_WINDOWS: - # Not implemented for windows - return self.old_pid = self.pid self.old_host = self.host self.pid = os.getpid() self.host = socket.gethostname() - # write pid, host to disk to sync over FS self._file.seek(0) self._file.write(f"pid={self.pid},host={self.host}".encode("utf-8")) @@ -514,6 +583,15 @@ def _write_log_debug_data(self) -> None: self._file.flush() os.fsync(self._file.fileno()) + def _release_lock(self): + if IS_WINDOWS: + hfile = win32file._get_osfhandle(self._file.fileno()) + overlapped = _setup_overlapped(self._start) + low_range, high_range = _low_high(self._length) + win32file.UnlockFileEx(hfile, low_range, high_range, overlapped) + else: + fcntl.lockf(self._file.fileno(), LockType.LOCK_UN, self._length, self._start, os.SEEK_SET) + def _unlock(self): """Releases a lock using POSIX locks (``fcntl.lockf``) @@ -524,11 +602,7 @@ def _unlock(self): """ assert self._file is not None, "cannot unlock without the file being set" - if IS_WINDOWS: - hfile = win32file._get_osfhandle(self._file.fileno()) - win32file.UnlockFileEx(hfile, 0, 0xFFFF0000, pywintypes.OVERLAPPED()) - else: - fcntl.lockf(self._file.fileno(), LockType.LOCK_UN, self._length, self._start, os.SEEK_SET) + self._release_lock() FILE_TRACKER.release_by_fh(self._file) self._file = None @@ -576,18 +650,19 @@ def acquire_write(self, timeout: Optional[float] = None) -> bool: timeout = timeout or self.default_timeout if self._writes == 0: - # can raise LockError. - wait_time, nattempts = self._lock(LockType.WRITE, timeout=timeout) - self._writes += 1 - self._current_lock = LockType.LOCK_EX - # Log if acquired, which includes counts when verbose - self._log_acquired("WRITE LOCK", wait_time, nattempts) - - # return True only if we weren't nested in a read lock. - # TODO: we may need to return two values: whether we got - # the write lock, and whether this is acquiring a read OR - # write lock for the first time. Now it returns the latter. - return self._reads == 0 + with _safe_exclusion(self): + # can raise LockError. + wait_time, nattempts = self._lock(LockType.WRITE, timeout=timeout) + self._writes += 1 + self._current_lock = LockType.LOCK_EX + # Log if acquired, which includes counts when verbose + self._log_acquired("WRITE LOCK", wait_time, nattempts) + + # return True only if we weren't nested in a read lock. + # TODO: we may need to return two values: whether we got + # the write lock, and whether this is acquiring a read OR + # write lock for the first time. Now it returns the latter. + return self._reads == 0 else: # Increment the write count for nested lock tracking self._writes += 1 @@ -620,12 +695,13 @@ def downgrade_write_to_read(self, timeout: Optional[float] = None) -> None: timeout = timeout or self.default_timeout if self._writes == 1: - self._log_downgrading() - # can raise LockError. - wait_time, nattempts = self._lock(LockType.READ, timeout=timeout) - self._reads = 1 - self._writes = 0 - self._log_downgraded(wait_time, nattempts) + with _safe_downgrade(self): + self._log_downgrading() + # can raise LockError. + wait_time, nattempts = self._lock(LockType.READ, timeout=timeout) + self._reads = 1 + self._writes = 0 + self._log_downgraded(wait_time, nattempts) else: raise LockDowngradeError(self.path) @@ -640,7 +716,7 @@ def upgrade_read_to_write(self, timeout: Optional[float] = None) -> None: if self._reads >= 1 and self._writes == 0: self._log_upgrading() - with _partial_upgrade(self.path): + with _safe_exclusion(self): # can raise LockError. wait_time, nattempts = self._lock(LockType.WRITE, timeout=timeout) self._reads = 0 diff --git a/lib/spack/spack/test/llnl/util/lock.py b/lib/spack/spack/test/llnl/util/lock.py index 06b180b185e797..52af6fa08601f4 100644 --- a/lib/spack/spack/test/llnl/util/lock.py +++ b/lib/spack/spack/test/llnl/util/lock.py @@ -295,7 +295,7 @@ def wait(self): # Process snippets below can be composed into tests. # class AcquireWrite: - def __init__(self, lock_path, start=0, length=0): + def __init__(self, lock_path, start=0, length=1): self.lock_path = lock_path self.start = start self.length = length @@ -312,7 +312,7 @@ def __call__(self, barrier): class AcquireRead: - def __init__(self, lock_path, start=0, length=0): + def __init__(self, lock_path, start=0, length=1): self.lock_path = lock_path self.start = start self.length = length @@ -329,7 +329,7 @@ def __call__(self, barrier): class TimeoutWrite: - def __init__(self, lock_path, start=0, length=0): + def __init__(self, lock_path, start=0, length=1): self.lock_path = lock_path self.start = start self.length = length @@ -347,7 +347,7 @@ def __call__(self, barrier): class TimeoutRead: - def __init__(self, lock_path, start=0, length=0): + def __init__(self, lock_path, start=0, length=1): self.lock_path = lock_path self.start = start self.length = length @@ -585,7 +585,7 @@ def test_write_lock_timeout_with_multiple_readers_3_2_ranges(lock_path): ) -@pytest.mark.skipif(getuid() == 0, reason="user is root") +@pytest.mark.skipif(sys.platform == "win32" or getuid() == 0, reason="user is root") def test_read_lock_on_read_only_lockfile(lock_dir, lock_path): """read-only directory, read-only lockfile.""" touch(lock_path) @@ -613,7 +613,8 @@ def test_read_lock_read_only_dir_writable_lockfile(lock_dir, lock_path): pass -@pytest.mark.skipif(False if sys.platform == "win32" else getuid() == 0, reason="user is root") +# skipping on Windows as spack cannot currently make directories read only +@pytest.mark.skipif(sys.platform == "win32" or getuid() == 0, reason="user is root") def test_read_lock_no_lockfile(lock_dir, lock_path): """read-only directory, no lockfile (so can't create).""" with read_only(lock_dir): @@ -1300,7 +1301,7 @@ def test_attempts_str(): def test_lock_str(): lock = lk.Lock("lockfile") lockstr = str(lock) - assert "lockfile[0:0]" in lockstr + assert f"lockfile[0:{lk.WHOLE_FILE_RANGE}]" in lockstr assert "timeout=None" in lockstr assert "#reads=0, #writes=0" in lockstr From 986498a294ca7aa2abe463abf8bfc3c2aa4fc53d Mon Sep 17 00:00:00 2001 From: John Parent Date: Tue, 6 Jan 2026 15:35:41 -0500 Subject: [PATCH 13/13] style Signed-off-by: John Parent --- lib/spack/spack/llnl/util/lock.py | 40 ++++++++++++-------------- lib/spack/spack/test/llnl/util/lock.py | 11 ++++--- 2 files changed, 23 insertions(+), 28 deletions(-) diff --git a/lib/spack/spack/llnl/util/lock.py b/lib/spack/spack/llnl/util/lock.py index 04396651bbb721..6253c81ca281b7 100644 --- a/lib/spack/spack/llnl/util/lock.py +++ b/lib/spack/spack/llnl/util/lock.py @@ -50,7 +50,7 @@ "CantCreateLockError", ] -WHOLE_FILE_RANGE = 0XFFFFFFFF if IS_WINDOWS else 0 +WHOLE_FILE_RANGE = 0xFFFFFFFF if IS_WINDOWS else 0 ReleaseFnType = Optional[Callable[[], bool]] @@ -192,8 +192,8 @@ def _attempts_str(wait_time, nattempts): def _low_high(value): - low = value & 0XFFFFFFFF - high = (value >> 32) & 0XFFFFFFFF + low = value & 0xFFFFFFFF + high = (value >> 32) & 0xFFFFFFFF return low, high @@ -212,7 +212,7 @@ def _safe_exclusion(lock: "Lock", timeout: Optional[float] = None): """Lock upgrade guard for Windows, designed to allow for lock upgrading which Windows file locks do not natively support Uses one additional lockfile as a "gate" for upgrades. - + If a process is attempting to upgrade from a read to a write it must first take a write lock on this intermediate file before releasing existing read lock and retaking a lock on the same @@ -223,13 +223,13 @@ def _safe_exclusion(lock: "Lock", timeout: Optional[float] = None): # probably something like foo.lock.lock timeout = timeout or lock.default_timeout lock_path = lock.path + ".lock" - l = Lock(lock_path, start=lock._start, length=lock._length) + lk = Lock(lock_path, start=lock._start, length=lock._length) # cache previous value for read locks _reads = lock._reads try: # don't use the acquire lock method to avoid # recursion - l._lock(LockType.WRITE, timeout=timeout) + lk._lock(LockType.WRITE, timeout=timeout) # lock gate acquired, drop read lock if there is one # cannot use release read as we need to drop # the read lock if there is one, not just decrement the nested @@ -237,7 +237,7 @@ def _safe_exclusion(lock: "Lock", timeout: Optional[float] = None): if _reads: lock._release_lock() yield - except LockError as e: + except LockError: # doesn't matter what the error was # if there was a read lock previously # restore it @@ -245,8 +245,8 @@ def _safe_exclusion(lock: "Lock", timeout: Optional[float] = None): lock._lock(LockType.READ, timeout=timeout) raise finally: - if l.is_write_locked(): - l.release_write() + if lk.is_write_locked(): + lk.release_write() @contextlib.contextmanager @@ -255,13 +255,13 @@ def _safe_downgrade(lock: "Lock", timeout: Optional[float] = None): if the exclusive lock is held first. When the shared lock overlaps the exclusive range, the exclusive lock must also be unlocked and must be unlocked *first* From the MSFT docs: - + A shared lock can overlap an exclusive lock if both locks were created using - the same file handle. If the same range is locked with an exclusive and a + the same file handle. If the same range is locked with an exclusive and a shared lock, two unlock operations are necessary to unlock the region; the first unlock operation unlocks the exclusive lock, the second unlock operation unlocks the shared lock. - + Since we are "downgrading" we no longer care to have an exclusive lock so we just give it up """ @@ -269,14 +269,13 @@ def _safe_downgrade(lock: "Lock", timeout: Optional[float] = None): # release the read lock first # so in the event on an error, we still have the exclusive lock yield - # From the docstring: + # From the docstring: # first unlock operation unlocks the exclusive lock, the second unlock # operation unlocks the shared lock. # # This will remove the exclusive lock lock._release_lock() - - + class LockType: READ = 0 @@ -439,7 +438,6 @@ def __str__(self) -> str: activity = "#reads={0}, #writes={1}".format(self._reads, self._writes) return "({0}, {1}, {2})".format(location, timeout, activity) - def _lock(self, op: int, timeout: Optional[float] = None) -> Tuple[float, int]: """This takes a lock using POSIX locks (``fcntl.lockf``). @@ -502,11 +500,7 @@ def _poll_lock(self, op: int) -> bool: range_low, range_high = _low_high(self._length) hfile = win32file._get_osfhandle(self._file.fileno()) win32file.LockFileEx( - hfile, - module_op | LockType.LOCK_NB, # flags - range_low, - range_high, - overlapped, + hfile, module_op | LockType.LOCK_NB, range_low, range_high, overlapped # flags ) else: # Try to get the lock (will raise if not available.) @@ -590,7 +584,9 @@ def _release_lock(self): low_range, high_range = _low_high(self._length) win32file.UnlockFileEx(hfile, low_range, high_range, overlapped) else: - fcntl.lockf(self._file.fileno(), LockType.LOCK_UN, self._length, self._start, os.SEEK_SET) + fcntl.lockf( + self._file.fileno(), LockType.LOCK_UN, self._length, self._start, os.SEEK_SET + ) def _unlock(self): """Releases a lock using POSIX locks (``fcntl.lockf``) diff --git a/lib/spack/spack/test/llnl/util/lock.py b/lib/spack/spack/test/llnl/util/lock.py index 52af6fa08601f4..3c86df6294fa84 100644 --- a/lib/spack/spack/test/llnl/util/lock.py +++ b/lib/spack/spack/test/llnl/util/lock.py @@ -1327,7 +1327,8 @@ def test_downgrade_write_fails(tmp_path: pathlib.Path): lock.downgrade_write_to_read() lock.release_read() -@pytest.mark.skipif(sys.platform=="win32", reason="fcntl unavailable on Windows") + +@pytest.mark.skipif(sys.platform == "win32", reason="fcntl unavailable on Windows") @pytest.mark.parametrize( "err_num,err_msg", [ @@ -1358,13 +1359,10 @@ def _lockf(fd, cmd, len, start, whence): monkeypatch.undo() lock.release_read() + @pytest.mark.skipif(sys.platform != "win32", reason="win32file only available on Windows") @pytest.mark.parametrize( - "err_num,err_msg", - [ - (32, "Fake EACCES error analog"), - (33, "Fake EAGAIN error analog"), - ], + "err_num,err_msg", [(32, "Fake EACCES error analog"), (33, "Fake EAGAIN error analog")] ) def test_poll_lock_exception_win(tmp_path: pathlib.Path, monkeypatch, err_num, err_msg): """Test poll lock exception handling.""" @@ -1388,6 +1386,7 @@ def LockFileEx(hfile, int_, int1_, int2_, ol): monkeypatch.undo() lock.release_read() + def test_upgrade_read_okay(tmp_path: pathlib.Path): """Test the lock read-to-write upgrade operation.""" with working_dir(str(tmp_path)):