From 71dca90cb924b5ef6364ff612a47c63849b08778 Mon Sep 17 00:00:00 2001 From: Everaldo Date: Wed, 6 Aug 2025 11:38:55 -0700 Subject: [PATCH 1/2] Improve routine lock. --- src/admin.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/admin.py b/src/admin.py index e15ca475..cf5f7b56 100644 --- a/src/admin.py +++ b/src/admin.py @@ -24,6 +24,7 @@ import json import logging +import os import random import time import zipfile @@ -280,13 +281,13 @@ def refresh_has_metakg(): refresh = refresh_document check = check_uptime -# only one process should perform backup routines -_lock = FileLock(".lock", timeout=0) - def routine(no_backup=False, format="zip"): logger = logging.getLogger("routine") + # only one process should perform backup routines + _lock = FileLock(".lock", timeout=0) + # Add jitter: random delay between 100 and 500 milliseconds (adjust range as needed) jitter_ms = random.uniform(100, 500) # Jitter in milliseconds jitter_seconds = jitter_ms / 1000 # Convert milliseconds to seconds @@ -326,6 +327,15 @@ def routine(no_backup=False, format="zip"): _lock.release() logger.info("Schedule lock released successfully.") + # Try to delete the .lock file manually if it still exists + lock_file_path = ".lock" + if os.path.exists(lock_file_path): + try: + os.remove(lock_file_path) + logger.info(".lock file manually deleted.") + except Exception as e: + logger.warning(f"Could not delete .lock file: {e}") + if __name__ == "__main__": restore_from_s3() From 129807ab1c8872dfa7f5db0449a94e6373c05796 Mon Sep 17 00:00:00 2001 From: Everaldo Date: Thu, 7 Aug 2025 12:26:23 -0700 Subject: [PATCH 2/2] Add PID to .lock file. --- src/admin.py | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/src/admin.py b/src/admin.py index cf5f7b56..73f48fad 100644 --- a/src/admin.py +++ b/src/admin.py @@ -275,6 +275,20 @@ def refresh_has_metakg(): smartapi.save() +def get_lock_pid(): + """Read PID from lock file if it exists""" + lock_file_path = ".lock" + if os.path.exists(lock_file_path): + try: + with open(lock_file_path, "r") as f: + pid = f.read().strip() + return int(pid) if pid.isdigit() else None + except Exception as e: + logging.warning(f"Could not read PID from {lock_file_path}: {e}") + return None + return None + + restore = restore_from_file backup = backup_to_file @@ -301,7 +315,11 @@ def routine(no_backup=False, format="zip"): # it won't block here lock_acquired = _lock.acquire() if lock_acquired: - logger.info("Schedule lock acquired successfully.") + # Write PID to the lock file + current_pid = os.getpid() + with open(".lock", "w") as lock_file: + lock_file.write(str(current_pid)) + logger.info(f"Schedule lock acquired successfully. PID {current_pid} written to lock file.") if not no_backup: logger.info(f"backup_to_s3(format={format})") backup_to_s3(format=format) @@ -318,7 +336,11 @@ def routine(no_backup=False, format="zip"): else: logger.warning("Schedule lock acquired by another process. No need to run it in this process.") except Timeout: - logger.warning("Schedule lock acquired by another process. No need to run it in this process.") + existing_pid = get_lock_pid() + if existing_pid: + logger.warning(f"Schedule lock acquired by another process (PID: {existing_pid}). No need to run it in this process.") + else: + logger.warning("Schedule lock acquired by another process. No need to run it in this process.") except Exception as e: logger.error(f"An error occurred during the routine: {e}") logger.error("Stack trace:", exc_info=True) @@ -331,8 +353,13 @@ def routine(no_backup=False, format="zip"): lock_file_path = ".lock" if os.path.exists(lock_file_path): try: + # Log the PID that was in the lock file before deletion + existing_pid = get_lock_pid() os.remove(lock_file_path) - logger.info(".lock file manually deleted.") + if existing_pid: + logger.info(f".lock file manually deleted (contained PID: {existing_pid}).") + else: + logger.info(".lock file manually deleted.") except Exception as e: logger.warning(f"Could not delete .lock file: {e}")