From fd7fce919a06fe2101cd20f41236653845c7d910 Mon Sep 17 00:00:00 2001 From: e-buerger Date: Wed, 4 May 2022 09:07:37 +0200 Subject: [PATCH 01/33] create classfiles --- exec_manager/ExecProfile.py | 8 ++++++++ exec_manager/ExecProfileType.py | 7 +++++++ exec_manager/Job.py | 34 +++++++++++++++++++++++++++++++ exec_manager/JobFactory.py | 9 ++++++++ exec_manager/JobStatusType.py | 12 +++++++++++ exec_manager/PythonJob.py | 23 +++++++++++++++++++++ exec_manager/WfLangType.py | 8 ++++++++ exec_manager/dao/JobDAO.py | 9 ++++++++ exec_manager/dao/JobFactoryDAO.py | 12 +++++++++++ 9 files changed, 122 insertions(+) create mode 100644 exec_manager/ExecProfile.py create mode 100644 exec_manager/ExecProfileType.py create mode 100644 exec_manager/Job.py create mode 100644 exec_manager/JobFactory.py create mode 100644 exec_manager/JobStatusType.py create mode 100644 exec_manager/PythonJob.py create mode 100644 exec_manager/WfLangType.py create mode 100644 exec_manager/dao/JobDAO.py create mode 100644 exec_manager/dao/JobFactoryDAO.py diff --git a/exec_manager/ExecProfile.py b/exec_manager/ExecProfile.py new file mode 100644 index 0000000..f073e7e --- /dev/null +++ b/exec_manager/ExecProfile.py @@ -0,0 +1,8 @@ +import ExecProfileType +import WfLangType + + +class ExecProfile: + def __init__(self, type: ExecProfileType, wf_lang: WfLangType) -> None: + self.type = type + self.wf_lang = wf_lang diff --git a/exec_manager/ExecProfileType.py b/exec_manager/ExecProfileType.py new file mode 100644 index 0000000..7f70619 --- /dev/null +++ b/exec_manager/ExecProfileType.py @@ -0,0 +1,7 @@ +from enum import Enum + + +class Exec_ProfileType(Enum): + BASH = "bash" + PYTHON = "python" + WES = "wes" diff --git a/exec_manager/Job.py b/exec_manager/Job.py new file mode 100644 index 0000000..4df9f13 --- /dev/null +++ b/exec_manager/Job.py @@ -0,0 +1,34 @@ +from abc import abstractmethod +from sys import exec_prefix +from uuid import uuid4 + +import JobStatusType +import ExecProfile + + +class Job: + def __init__(self, job_id: uuid4, job_status: JobStatusType, exec_profile: ExecProfile) -> None: + self.job_id = job_id + self.job_status=job_status + self.exec_profile=exec_profile + + @abstractmethod + def prepare() -> None: + """This method should implement the prepare execution step""" + + @abstractmethod + def exec() -> None: + """This method should implement the exec execution step""" + + @abstractmethod + def eval() -> None: + """This method should implement the eval execution step""" + + @abstractmethod + def finalize() -> None: + """This method should implement the finalize execution step""" + + @abstractmethod + def cancel() -> None: + """This method should implement the cancel execution step""" + \ No newline at end of file diff --git a/exec_manager/JobFactory.py b/exec_manager/JobFactory.py new file mode 100644 index 0000000..985bb76 --- /dev/null +++ b/exec_manager/JobFactory.py @@ -0,0 +1,9 @@ +import ExecProfile +import Job + +class JobFactory: + def __init__(self) -> None: + pass + + def create_job(self, inputs: dict, workflow, exec_profile: ExecProfile) -> Job: + pass \ No newline at end of file diff --git a/exec_manager/JobStatusType.py b/exec_manager/JobStatusType.py new file mode 100644 index 0000000..96d6ec2 --- /dev/null +++ b/exec_manager/JobStatusType.py @@ -0,0 +1,12 @@ +from enum import Enum + + +class JobStatusType(Enum): + NOTSTARTET = "not startetd" + PREPARING = "preparing" + EXECUTING = "executing" + EVALUATING = "evaluating" + FINALZING = "finalizing" + CANCELED = "canceled" + FAILED = "failed" + SUCCEEDED = "succeeded" diff --git a/exec_manager/PythonJob.py b/exec_manager/PythonJob.py new file mode 100644 index 0000000..4d8794f --- /dev/null +++ b/exec_manager/PythonJob.py @@ -0,0 +1,23 @@ +from uuid import uuid4 +import Job +import ExecProfile +import JobStatusType + +class PythonJob(Job): + def __init__(self, job_id: uuid4, job_status: JobStatusType, exec_profile: ExecProfile) -> None: + super().__init__(job_id, job_status, exec_profile) + + def prepare() -> None: + pass + + def exec() -> None: + pass + + def eval() -> None: + pass + + def finalize() -> None: + pass + + def cancel() -> None: + pass \ No newline at end of file diff --git a/exec_manager/WfLangType.py b/exec_manager/WfLangType.py new file mode 100644 index 0000000..5416282 --- /dev/null +++ b/exec_manager/WfLangType.py @@ -0,0 +1,8 @@ +from enum import Enum + + +class WfLangType(Enum): + CWL = "cwl" + WDL = "wdl" + NEXTFLOW = "nextflow" + SNAKEMAKE = "snakemake" diff --git a/exec_manager/dao/JobDAO.py b/exec_manager/dao/JobDAO.py new file mode 100644 index 0000000..8446cc0 --- /dev/null +++ b/exec_manager/dao/JobDAO.py @@ -0,0 +1,9 @@ +from uuid import uuid4 +import JobStatusType + +class JobDAO: + def __init__(self) -> None: + pass + + def update_job_status(self, job_id: uuid4, new_job_status: JobStatusType) -> None: + pass \ No newline at end of file diff --git a/exec_manager/dao/JobFactoryDAO.py b/exec_manager/dao/JobFactoryDAO.py new file mode 100644 index 0000000..a6e5dc1 --- /dev/null +++ b/exec_manager/dao/JobFactoryDAO.py @@ -0,0 +1,12 @@ +from uuid import uuid4 +import JobStatusType +import WfLangType +import ExecProfile + + +class JobFactoryDAO: + def __init__(self) -> None: + pass + + def create_job(job_status: JobStatusType, inputs: dict, workflow, wf_lang: WfLangType, exec_profile: ExecProfile) -> uuid4: + pass \ No newline at end of file From dad43725a5f6780807baca64d12f0b7302a0aa00 Mon Sep 17 00:00:00 2001 From: e-buerger Date: Wed, 4 May 2022 10:11:40 +0200 Subject: [PATCH 02/33] adapting code for passing precommit --- exec_manager/ExecProfile.py | 8 ----- exec_manager/JobFactory.py | 9 ----- exec_manager/PythonJob.py | 23 ------------ exec_manager/dao/JobDAO.py | 9 ----- exec_manager/dao/JobFactoryDAO.py | 12 ------- exec_manager/dao/job_dao.py | 11 ++++++ exec_manager/dao/job_factory_dao.py | 18 ++++++++++ exec_manager/exec_profile.py | 8 +++++ ...xecProfileType.py => exec_profile_type.py} | 0 exec_manager/{Job.py => job.py} | 36 +++++++++---------- exec_manager/job_factory.py | 10 ++++++ .../{JobStatusType.py => job_status_type.py} | 0 exec_manager/python_job.py | 27 ++++++++++++++ .../{WfLangType.py => wf_lang_type.py} | 0 14 files changed, 92 insertions(+), 79 deletions(-) delete mode 100644 exec_manager/ExecProfile.py delete mode 100644 exec_manager/JobFactory.py delete mode 100644 exec_manager/PythonJob.py delete mode 100644 exec_manager/dao/JobDAO.py delete mode 100644 exec_manager/dao/JobFactoryDAO.py create mode 100644 exec_manager/dao/job_dao.py create mode 100644 exec_manager/dao/job_factory_dao.py create mode 100644 exec_manager/exec_profile.py rename exec_manager/{ExecProfileType.py => exec_profile_type.py} (100%) rename exec_manager/{Job.py => job.py} (52%) create mode 100644 exec_manager/job_factory.py rename exec_manager/{JobStatusType.py => job_status_type.py} (100%) create mode 100644 exec_manager/python_job.py rename exec_manager/{WfLangType.py => wf_lang_type.py} (100%) diff --git a/exec_manager/ExecProfile.py b/exec_manager/ExecProfile.py deleted file mode 100644 index f073e7e..0000000 --- a/exec_manager/ExecProfile.py +++ /dev/null @@ -1,8 +0,0 @@ -import ExecProfileType -import WfLangType - - -class ExecProfile: - def __init__(self, type: ExecProfileType, wf_lang: WfLangType) -> None: - self.type = type - self.wf_lang = wf_lang diff --git a/exec_manager/JobFactory.py b/exec_manager/JobFactory.py deleted file mode 100644 index 985bb76..0000000 --- a/exec_manager/JobFactory.py +++ /dev/null @@ -1,9 +0,0 @@ -import ExecProfile -import Job - -class JobFactory: - def __init__(self) -> None: - pass - - def create_job(self, inputs: dict, workflow, exec_profile: ExecProfile) -> Job: - pass \ No newline at end of file diff --git a/exec_manager/PythonJob.py b/exec_manager/PythonJob.py deleted file mode 100644 index 4d8794f..0000000 --- a/exec_manager/PythonJob.py +++ /dev/null @@ -1,23 +0,0 @@ -from uuid import uuid4 -import Job -import ExecProfile -import JobStatusType - -class PythonJob(Job): - def __init__(self, job_id: uuid4, job_status: JobStatusType, exec_profile: ExecProfile) -> None: - super().__init__(job_id, job_status, exec_profile) - - def prepare() -> None: - pass - - def exec() -> None: - pass - - def eval() -> None: - pass - - def finalize() -> None: - pass - - def cancel() -> None: - pass \ No newline at end of file diff --git a/exec_manager/dao/JobDAO.py b/exec_manager/dao/JobDAO.py deleted file mode 100644 index 8446cc0..0000000 --- a/exec_manager/dao/JobDAO.py +++ /dev/null @@ -1,9 +0,0 @@ -from uuid import uuid4 -import JobStatusType - -class JobDAO: - def __init__(self) -> None: - pass - - def update_job_status(self, job_id: uuid4, new_job_status: JobStatusType) -> None: - pass \ No newline at end of file diff --git a/exec_manager/dao/JobFactoryDAO.py b/exec_manager/dao/JobFactoryDAO.py deleted file mode 100644 index a6e5dc1..0000000 --- a/exec_manager/dao/JobFactoryDAO.py +++ /dev/null @@ -1,12 +0,0 @@ -from uuid import uuid4 -import JobStatusType -import WfLangType -import ExecProfile - - -class JobFactoryDAO: - def __init__(self) -> None: - pass - - def create_job(job_status: JobStatusType, inputs: dict, workflow, wf_lang: WfLangType, exec_profile: ExecProfile) -> uuid4: - pass \ No newline at end of file diff --git a/exec_manager/dao/job_dao.py b/exec_manager/dao/job_dao.py new file mode 100644 index 0000000..14a0439 --- /dev/null +++ b/exec_manager/dao/job_dao.py @@ -0,0 +1,11 @@ +import uuid + +import exec_manager.job_status_type as job_status_type + + +class JobDAO: + def __init__(self) -> None: + pass + + def update_job_status(self, job_id: uuid, new_job_status: job_status_type) -> None: + pass diff --git a/exec_manager/dao/job_factory_dao.py b/exec_manager/dao/job_factory_dao.py new file mode 100644 index 0000000..3930b2e --- /dev/null +++ b/exec_manager/dao/job_factory_dao.py @@ -0,0 +1,18 @@ +import uuid + +import exec_manager.exec_profile as exec_profile +import exec_manager.job_status_type as job_status_type + + +class JobFactoryDAO: + def __init__(self) -> None: + pass + + def create_job( + self, + job_status: job_status_type, + inputs: dict, + workflow, + exec_profile: exec_profile, + ) -> uuid: + pass diff --git a/exec_manager/exec_profile.py b/exec_manager/exec_profile.py new file mode 100644 index 0000000..08d5dcf --- /dev/null +++ b/exec_manager/exec_profile.py @@ -0,0 +1,8 @@ +import exec_manager.exec_profile_type as exec_profile_type +import exec_manager.wf_lang_type as wf_lang_type + + +class ExecProfile: + def __init__(self, type: exec_profile_type, wf_lang: wf_lang_type) -> None: + self.type = type + self.wf_lang = wf_lang diff --git a/exec_manager/ExecProfileType.py b/exec_manager/exec_profile_type.py similarity index 100% rename from exec_manager/ExecProfileType.py rename to exec_manager/exec_profile_type.py diff --git a/exec_manager/Job.py b/exec_manager/job.py similarity index 52% rename from exec_manager/Job.py rename to exec_manager/job.py index 4df9f13..a7e8eba 100644 --- a/exec_manager/Job.py +++ b/exec_manager/job.py @@ -1,34 +1,34 @@ +import uuid from abc import abstractmethod -from sys import exec_prefix -from uuid import uuid4 -import JobStatusType -import ExecProfile +import exec_manager.exec_profile as exec_profile +import exec_manager.job_status_type as job_status_type class Job: - def __init__(self, job_id: uuid4, job_status: JobStatusType, exec_profile: ExecProfile) -> None: + def __init__( + self, job_id: uuid, job_status: job_status_type, exec_profile: exec_profile + ) -> None: self.job_id = job_id - self.job_status=job_status - self.exec_profile=exec_profile - + self.job_status = job_status + self.exec_profile = exec_profile + @abstractmethod - def prepare() -> None: + def prepare(self) -> None: """This method should implement the prepare execution step""" - + @abstractmethod - def exec() -> None: + def exec(self) -> None: """This method should implement the exec execution step""" - + @abstractmethod - def eval() -> None: + def eval(self) -> None: """This method should implement the eval execution step""" - + @abstractmethod - def finalize() -> None: + def finalize(self) -> None: """This method should implement the finalize execution step""" - + @abstractmethod - def cancel() -> None: + def cancel(self) -> None: """This method should implement the cancel execution step""" - \ No newline at end of file diff --git a/exec_manager/job_factory.py b/exec_manager/job_factory.py new file mode 100644 index 0000000..837d1c2 --- /dev/null +++ b/exec_manager/job_factory.py @@ -0,0 +1,10 @@ +import exec_manager.exec_profile as exec_profile +import exec_manager.job as job + + +class JobFactory: + def __init__(self) -> None: + pass + + def create_job(self, inputs: dict, workflow, exec_profile: exec_profile) -> job: + pass diff --git a/exec_manager/JobStatusType.py b/exec_manager/job_status_type.py similarity index 100% rename from exec_manager/JobStatusType.py rename to exec_manager/job_status_type.py diff --git a/exec_manager/python_job.py b/exec_manager/python_job.py new file mode 100644 index 0000000..8193621 --- /dev/null +++ b/exec_manager/python_job.py @@ -0,0 +1,27 @@ +import uuid + +import exec_manager.exec_profile as exec_profile +import exec_manager.job as job +import exec_manager.job_status_type as job_status_type + + +class PythonJob(job): + def __init__( + self, job_id: uuid, job_status: job_status_type, exec_profile: exec_profile + ) -> None: + super().__init__(job_id, job_status, exec_profile) + + def prepare(self) -> None: + pass + + def exec(self) -> None: + pass + + def eval(self) -> None: + pass + + def finalize(self) -> None: + pass + + def cancel(self) -> None: + pass diff --git a/exec_manager/WfLangType.py b/exec_manager/wf_lang_type.py similarity index 100% rename from exec_manager/WfLangType.py rename to exec_manager/wf_lang_type.py From f59a516c609d56c27eb96429dbfb049fe1a3bcc0 Mon Sep 17 00:00:00 2001 From: e-buerger Date: Wed, 4 May 2022 11:19:31 +0200 Subject: [PATCH 03/33] adapt code for passing precommit --- exec_manager/__init__.py | 2 +- exec_manager/dao/job_dao.py | 26 +++++++++++--- exec_manager/dao/job_factory_dao.py | 38 +++++++++++++++----- exec_manager/exec_profile.py | 37 ++++++++++++++++--- exec_manager/exec_profile_type.py | 11 +++++- exec_manager/job.py | 50 +++++++++++++++++++++++--- exec_manager/job_factory.py | 27 +++++++++++--- exec_manager/job_status_type.py | 19 ++++++++++ exec_manager/python_job.py | 56 +++++++++++++++++++++-------- exec_manager/wf_lang_type.py | 11 ++++++ 10 files changed, 235 insertions(+), 42 deletions(-) diff --git a/exec_manager/__init__.py b/exec_manager/__init__.py index 7589079..0921524 100644 --- a/exec_manager/__init__.py +++ b/exec_manager/__init__.py @@ -13,6 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Short description of package""" # Please adapt to package +"""backend""" # Please adapt to package __version__ = "0.1.0" diff --git a/exec_manager/dao/job_dao.py b/exec_manager/dao/job_dao.py index 14a0439..2e29277 100644 --- a/exec_manager/dao/job_dao.py +++ b/exec_manager/dao/job_dao.py @@ -1,11 +1,27 @@ -import uuid +"""class for job dao""" -import exec_manager.job_status_type as job_status_type +from uuid import UUID + +from exec_manager.job_status_type import JobStatusType class JobDAO: + """ + class for job dao + + ... + + Attributes + ---------- + + Methods + ------- + update_job_status(self, job_id: UUID, new_job_status: JobStatusType) -> None: + updates the status of the job in database + """ + def __init__(self) -> None: - pass + """constructor""" - def update_job_status(self, job_id: uuid, new_job_status: job_status_type) -> None: - pass + def update_job_status(self, job_id: UUID, new_job_status: JobStatusType) -> None: + """this method updates the job status""" diff --git a/exec_manager/dao/job_factory_dao.py b/exec_manager/dao/job_factory_dao.py index 3930b2e..8d45b4d 100644 --- a/exec_manager/dao/job_factory_dao.py +++ b/exec_manager/dao/job_factory_dao.py @@ -1,18 +1,40 @@ -import uuid +"""class for job factory dao""" -import exec_manager.exec_profile as exec_profile -import exec_manager.job_status_type as job_status_type +from uuid import UUID + +from exec_manager.exec_profile import ExecProfile +from exec_manager.job_status_type import JobStatusType class JobFactoryDAO: + """ + class for job factory dao + + ... + + Attributes + ---------- + + Methods + ------- + create_job( + self, + job_status: JobStatusType, + inputs: dict, + workflow, + exec_profile: ExecProfile, + ) -> uuid: + inserts job into database + """ + def __init__(self) -> None: - pass + """constructor""" def create_job( self, - job_status: job_status_type, + job_status: JobStatusType, inputs: dict, workflow, - exec_profile: exec_profile, - ) -> uuid: - pass + exec_profile: ExecProfile, + ) -> UUID: + """this method creates a job""" diff --git a/exec_manager/exec_profile.py b/exec_manager/exec_profile.py index 08d5dcf..c95f15c 100644 --- a/exec_manager/exec_profile.py +++ b/exec_manager/exec_profile.py @@ -1,8 +1,37 @@ -import exec_manager.exec_profile_type as exec_profile_type -import exec_manager.wf_lang_type as wf_lang_type +"""class for exec profile""" + +from exec_manager.exec_profile_type import ExecProfileType +from exec_manager.wf_lang_type import WfLangType class ExecProfile: - def __init__(self, type: exec_profile_type, wf_lang: wf_lang_type) -> None: - self.type = type + """ + class for Exec-Profile + + ... + + Attributes + ---------- + exec_profile_type : ExecProfileType + type of the exec profile (bash, python, wes) + wf_lang : WfLangType + workflow language (cwl, wdl, nextflow, snakemake + + Methods + ------- + """ + + def __init__(self, exec_profile_type: ExecProfileType, wf_lang: WfLangType) -> None: + """ + Constructs all the necessary attributes for the exec profile object. + + Parameters + ---------- + exec_profile_type : ExecProfileType + type of the exec profile (bash, python, wes) + wf_lang : WfLangType + workflow language (cwl, wdl, nextflow, snakemake + """ + + self.exec_profile_type = exec_profile_type self.wf_lang = wf_lang diff --git a/exec_manager/exec_profile_type.py b/exec_manager/exec_profile_type.py index 7f70619..165d43f 100644 --- a/exec_manager/exec_profile_type.py +++ b/exec_manager/exec_profile_type.py @@ -1,7 +1,16 @@ +"""enum for exec profile type""" + from enum import Enum -class Exec_ProfileType(Enum): +class ExecProfileType(Enum): + """enumaerate exec profile types""" + BASH = "bash" + """execution with bash""" + PYTHON = "python" + """execution with python""" + WES = "wes" + """execution with wes""" diff --git a/exec_manager/job.py b/exec_manager/job.py index a7e8eba..1e08756 100644 --- a/exec_manager/job.py +++ b/exec_manager/job.py @@ -1,14 +1,56 @@ -import uuid +"""class for job""" + from abc import abstractmethod +from uuid import UUID -import exec_manager.exec_profile as exec_profile -import exec_manager.job_status_type as job_status_type +from exec_manager.exec_profile import ExecProfile +from exec_manager.job_status_type import JobStatusType class Job: + """ + class for job + + ... + + Attributes + ---------- + job_id : uuid + id of the job + job_status : JobStatusType + current status of the job (eg. notstarted, succeeded, failed) + exec_profile : ExecProfile + exec profile with which the job should be executed + + Methods + ------- + prepare() -> None: + prepares the job + exec() -> None: + executes the job + eval() -> None: + evaluates the job + finalize() -> None: + finalizes the job + cancel() -> None: + cancels the job + """ + def __init__( - self, job_id: uuid, job_status: job_status_type, exec_profile: exec_profile + self, job_id: UUID, job_status: JobStatusType, exec_profile: ExecProfile ) -> None: + """ + Constructs all the necessary attributes for the job object. + + Parameters + ---------- + job_id : uuid + id of the job + job_status : JobStatusType + current status of the job (eg. notstarted, succeeded, failed) + exec_profile : ExecProfile + exec profile with which the job should be executed + """ self.job_id = job_id self.job_status = job_status self.exec_profile = exec_profile diff --git a/exec_manager/job_factory.py b/exec_manager/job_factory.py index 837d1c2..abb1229 100644 --- a/exec_manager/job_factory.py +++ b/exec_manager/job_factory.py @@ -1,10 +1,27 @@ -import exec_manager.exec_profile as exec_profile -import exec_manager.job as job +"""class for job factory""" + +import job + +from exec_manager.exec_profile import ExecProfile class JobFactory: + """ + class for job factory + + ... + + Attributes + ---------- + + Methods + ------- + create_job(inputs: dict, workflow, exec_profile: ExecProfile) -> job: + creates a new job + """ + def __init__(self) -> None: - pass + """this is the constructor""" - def create_job(self, inputs: dict, workflow, exec_profile: exec_profile) -> job: - pass + def create_job(self, inputs: dict, workflow, exec_profile: ExecProfile) -> job: + """this method creates a new job""" diff --git a/exec_manager/job_status_type.py b/exec_manager/job_status_type.py index 96d6ec2..2b33bfc 100644 --- a/exec_manager/job_status_type.py +++ b/exec_manager/job_status_type.py @@ -1,12 +1,31 @@ +"""enum for job status type""" + from enum import Enum class JobStatusType(Enum): + """enumerate job status types""" + NOTSTARTET = "not startetd" + """job ist not started yet""" + PREPARING = "preparing" + """job is preparing""" + EXECUTING = "executing" + """job is executing""" + EVALUATING = "evaluating" + """job ist evaluating""" + FINALZING = "finalizing" + """job is finalizing""" + CANCELED = "canceled" + """job is canceled""" + FAILED = "failed" + """job is failed""" + SUCCEEDED = "succeeded" + """job is succeeded""" diff --git a/exec_manager/python_job.py b/exec_manager/python_job.py index 8193621..57b332a 100644 --- a/exec_manager/python_job.py +++ b/exec_manager/python_job.py @@ -1,27 +1,55 @@ -import uuid +"""class for python job""" + +from uuid import UUID + +from exec_manager.exec_profile import ExecProfile +from exec_manager.job_status_type import JobStatusType + + +class PythonJob: + """ + class for python job + + ... + + Attributes + ---------- + job_id : uuid + id of the job + job_status : JobStatusType + current status of the job (eg. notstarted, succeeded, failed) + exec_profile : ExecProfile + python exec profile + + Methods + ------- + prepare() -> None: + prepares the job + exec() -> None: + executes the job + eval() -> None: + evaluates the job + finalize() -> None: + finalizes the job + cancel() -> None: + """ -import exec_manager.exec_profile as exec_profile -import exec_manager.job as job -import exec_manager.job_status_type as job_status_type - - -class PythonJob(job): def __init__( - self, job_id: uuid, job_status: job_status_type, exec_profile: exec_profile + self, job_id: UUID, job_status: JobStatusType, exec_profile: ExecProfile ) -> None: - super().__init__(job_id, job_status, exec_profile) + """constructor""" def prepare(self) -> None: - pass + """job preparation""" def exec(self) -> None: - pass + """job execution""" def eval(self) -> None: - pass + "job evaluation" def finalize(self) -> None: - pass + """job finalization""" def cancel(self) -> None: - pass + """job canceling""" diff --git a/exec_manager/wf_lang_type.py b/exec_manager/wf_lang_type.py index 5416282..25cb214 100644 --- a/exec_manager/wf_lang_type.py +++ b/exec_manager/wf_lang_type.py @@ -1,8 +1,19 @@ +"""enum for workflow languagae type""" + from enum import Enum class WfLangType(Enum): + """enumerate workflow language types""" + CWL = "cwl" + """cwl language""" + WDL = "wdl" + """wdl language""" + NEXTFLOW = "nextflow" + """nextflow language""" + SNAKEMAKE = "snakemake" + """snakemake language""" From a19747cdfe271a91c8ef8cd91867fe6c6910752a Mon Sep 17 00:00:00 2001 From: e-buerger Date: Wed, 4 May 2022 11:23:25 +0200 Subject: [PATCH 04/33] adapt code for passing precommit --- exec_manager/job_factory.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/exec_manager/job_factory.py b/exec_manager/job_factory.py index abb1229..8a3769b 100644 --- a/exec_manager/job_factory.py +++ b/exec_manager/job_factory.py @@ -1,8 +1,7 @@ """class for job factory""" -import job - from exec_manager.exec_profile import ExecProfile +from exec_manager.job import Job class JobFactory: @@ -16,12 +15,12 @@ class for job factory Methods ------- - create_job(inputs: dict, workflow, exec_profile: ExecProfile) -> job: + create_job(inputs: dict, workflow, exec_profile: ExecProfile) -> Job: creates a new job """ def __init__(self) -> None: """this is the constructor""" - def create_job(self, inputs: dict, workflow, exec_profile: ExecProfile) -> job: + def create_job(self, inputs: dict, workflow, exec_profile: ExecProfile) -> Job: """this method creates a new job""" From 7588dde1e8c9d02306508ea11947973c30bf2e01 Mon Sep 17 00:00:00 2001 From: e-buerger Date: Thu, 5 May 2022 09:55:31 +0200 Subject: [PATCH 05/33] implement a few functions --- exec_manager/dao/db_models.py | 27 +++---- exec_manager/dao/job_dao.py | 115 +++++++++++++++++++++++++++- exec_manager/dao/job_factory_dao.py | 40 ---------- exec_manager/exec_profile.py | 8 +- exec_manager/job.py | 18 ++--- exec_manager/job_factory.py | 27 ++++++- exec_manager/python_job.py | 79 +++++++++++++++++-- 7 files changed, 229 insertions(+), 85 deletions(-) delete mode 100644 exec_manager/dao/job_factory_dao.py diff --git a/exec_manager/dao/db_models.py b/exec_manager/dao/db_models.py index 805180e..a79a7ac 100644 --- a/exec_manager/dao/db_models.py +++ b/exec_manager/dao/db_models.py @@ -15,26 +15,21 @@ """Defines all database specific ORM models""" -from sqlalchemy import JSON, Boolean, Column, Integer, String +from sqlalchemy import JSON, Column, Integer, String from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm.decl_api import DeclarativeMeta -Base: DeclarativeMeta = declarative_base() - +from exec_manager.exec_profile import ExecProfile -class ExampleObjectA(Base): - """An example object stored in the DB""" - - __tablename__ = "visas" - id = Column(Integer, primary_key=True) - name = Column(String, nullable=False) - some_json_details = Column(JSON, nullable=False) +Base: DeclarativeMeta = declarative_base() -class ExampleObjectB(Base): - """Another example object stored in the DB""" +class DBJob(Base): + """An job object stored in the DB""" - __tablename__ = "table_b" - id = Column(Integer, primary_key=True) - name = Column(String, nullable=False) - active = Column(Boolean, nullable=False) + __tablename__ = "job" + job_id = Column(Integer, primary_key=True) + job_status = Column(String, nullable=False) + exec_profile = Column(ExecProfile, nullable=False) + workflow = Column(JSON, nullable=False) + inputs = Column(dict, nullable=False) diff --git a/exec_manager/dao/job_dao.py b/exec_manager/dao/job_dao.py index 2e29277..a44f96a 100644 --- a/exec_manager/dao/job_dao.py +++ b/exec_manager/dao/job_dao.py @@ -1,7 +1,13 @@ """class for job dao""" -from uuid import UUID +from uuid import UUID, uuid4 +from sqlalchemy import engine_from_config +from sqlalchemy.orm import Session + +from exec_manager.dao.db_models import DBJob +from exec_manager.exec_profile import ExecProfile +from exec_manager.job import Job from exec_manager.job_status_type import JobStatusType @@ -16,12 +22,115 @@ class for job dao Methods ------- + create_job( + self, + job_status: JobStatusType, + inputs: dict, + workflow, + exec_profile: ExecProfile, + ) -> UUID: + creates a job + update_job_status(self, job_id: UUID, new_job_status: JobStatusType) -> None: updates the status of the job in database + + get_job(job_id: UUID) -> Job: + returns a job by the job id + + generate_job_id() -> UUID: + generates a uuid as job id and checks its uniqueness """ def __init__(self) -> None: """constructor""" - def update_job_status(self, job_id: UUID, new_job_status: JobStatusType) -> None: - """this method updates the job status""" + +def create_job_dao( + job_status: JobStatusType, + exec_profile: ExecProfile, + workflow, + inputs: dict, +) -> UUID: + """ + Inserts a job into the database. + + Parameters + ---------- + job_status: JobStatusType + current status of the job; initially it is JobStatusType.NOTSTARTED + exec_profile: ExecProfile + exec profile of this job + workflow + the jobs workflow + inputs: dict + the input parameters of the job + + Returns + ------- + UUID + """ + + job_id = generate_job_id() + with Session(engine_from_config) as session: + with session.begin(): + session.insert(DBJob).values( + job_id, job_status, exec_profile, workflow, inputs + ) + return job_id + + +def generate_job_id() -> UUID: + """ + Generates an unique job id. + + Parameters + ---------- + + Returns + ------- + UUID + """ + job_id = uuid4() + while get_job(job_id) is not None: + job_id = uuid4() + return job_id + + +def update_job_status(job_id: UUID, new_job_status: JobStatusType) -> None: + """ + Updates a jobs status by his job id. + + Parameters + ---------- + job_id: UUID + id of the job + new_job_status: JobStatusType + new status of the job; cannot be JobStatusType.NOTSTARTED + + Returns + ------- + None + """ + with Session(engine_from_config) as session: + with session.begin(): + session.update(DBJob).where(DBJob.job_id == job_id).values( + job_status=new_job_status + ) + + +def get_job(job_id: UUID) -> Job: + """ + Returns a job by his job id. + + Parameters + ---------- + job_id: UUID + id of the job + + Returns + ------- + Job + """ + with Session(engine_from_config) as session: + with session.begin(): + return session.query(DBJob).filter_by(job_id=job_id).all() diff --git a/exec_manager/dao/job_factory_dao.py b/exec_manager/dao/job_factory_dao.py deleted file mode 100644 index 8d45b4d..0000000 --- a/exec_manager/dao/job_factory_dao.py +++ /dev/null @@ -1,40 +0,0 @@ -"""class for job factory dao""" - -from uuid import UUID - -from exec_manager.exec_profile import ExecProfile -from exec_manager.job_status_type import JobStatusType - - -class JobFactoryDAO: - """ - class for job factory dao - - ... - - Attributes - ---------- - - Methods - ------- - create_job( - self, - job_status: JobStatusType, - inputs: dict, - workflow, - exec_profile: ExecProfile, - ) -> uuid: - inserts job into database - """ - - def __init__(self) -> None: - """constructor""" - - def create_job( - self, - job_status: JobStatusType, - inputs: dict, - workflow, - exec_profile: ExecProfile, - ) -> UUID: - """this method creates a job""" diff --git a/exec_manager/exec_profile.py b/exec_manager/exec_profile.py index c95f15c..66669ae 100644 --- a/exec_manager/exec_profile.py +++ b/exec_manager/exec_profile.py @@ -27,10 +27,10 @@ def __init__(self, exec_profile_type: ExecProfileType, wf_lang: WfLangType) -> N Parameters ---------- - exec_profile_type : ExecProfileType - type of the exec profile (bash, python, wes) - wf_lang : WfLangType - workflow language (cwl, wdl, nextflow, snakemake + exec_profile_type : ExecProfileType + type of the exec profile (bash, python, wes) + wf_lang : WfLangType + workflow language (cwl, wdl, nextflow, snakemake """ self.exec_profile_type = exec_profile_type diff --git a/exec_manager/job.py b/exec_manager/job.py index 1e08756..c958df3 100644 --- a/exec_manager/job.py +++ b/exec_manager/job.py @@ -1,6 +1,5 @@ """class for job""" -from abc import abstractmethod from uuid import UUID from exec_manager.exec_profile import ExecProfile @@ -44,33 +43,28 @@ def __init__( Parameters ---------- - job_id : uuid - id of the job - job_status : JobStatusType - current status of the job (eg. notstarted, succeeded, failed) - exec_profile : ExecProfile - exec profile with which the job should be executed + job_id : uuid + id of the job + job_status : JobStatusType + current status of the job (eg. notstarted, succeeded, failed) + exec_profile : ExecProfile + exec profile with which the job should be executed """ self.job_id = job_id self.job_status = job_status self.exec_profile = exec_profile - @abstractmethod def prepare(self) -> None: """This method should implement the prepare execution step""" - @abstractmethod def exec(self) -> None: """This method should implement the exec execution step""" - @abstractmethod def eval(self) -> None: """This method should implement the eval execution step""" - @abstractmethod def finalize(self) -> None: """This method should implement the finalize execution step""" - @abstractmethod def cancel(self) -> None: """This method should implement the cancel execution step""" diff --git a/exec_manager/job_factory.py b/exec_manager/job_factory.py index 8a3769b..23b9372 100644 --- a/exec_manager/job_factory.py +++ b/exec_manager/job_factory.py @@ -1,7 +1,11 @@ """class for job factory""" +from exec_manager.dao.job_dao import create_job_dao from exec_manager.exec_profile import ExecProfile +from exec_manager.exec_profile_type import ExecProfileType from exec_manager.job import Job +from exec_manager.job_status_type import JobStatusType +from exec_manager.python_job import PythonJob class JobFactory: @@ -22,5 +26,24 @@ class for job factory def __init__(self) -> None: """this is the constructor""" - def create_job(self, inputs: dict, workflow, exec_profile: ExecProfile) -> Job: - """this method creates a new job""" + +def create_job(inputs: dict, workflow, exec_profile: ExecProfile) -> Job: + """ + Creates a job. + + Parameters + ---------- + + Returns + ------- + Job + """ + job_status = JobStatusType.NOTSTARTET + job_id = create_job_dao(job_status, exec_profile, workflow, inputs) + if exec_profile.exec_profile_type == ExecProfileType.PYTHON: + return PythonJob(job_id, job_status, exec_profile, inputs) + if exec_profile.exec_profile_type == ExecProfileType.BASH: + pass # place for bash job + if exec_profile.exec_profile_type == ExecProfileType.WES: + pass # place for wes job + return Job(job_id, job_status, exec_profile) diff --git a/exec_manager/python_job.py b/exec_manager/python_job.py index 57b332a..08c7540 100644 --- a/exec_manager/python_job.py +++ b/exec_manager/python_job.py @@ -2,11 +2,13 @@ from uuid import UUID +from exec_manager.dao.job_dao import update_job_status from exec_manager.exec_profile import ExecProfile +from exec_manager.job import Job from exec_manager.job_status_type import JobStatusType -class PythonJob: +class PythonJob(Job): """ class for python job @@ -35,21 +37,82 @@ class for python job """ def __init__( - self, job_id: UUID, job_status: JobStatusType, exec_profile: ExecProfile + self, + job_id: UUID, + job_status: JobStatusType, + exec_profile: ExecProfile, + inputs: dict, ) -> None: - """constructor""" + """Constructs all the necessary attributes for the python job object.""" + Job.__init__(self, job_id, job_status, exec_profile) + self.inputs = inputs def prepare(self) -> None: - """job preparation""" + """ + Prepares the job. + + Parameters + ---------- + + Returns + ------- + NONE + """ + # JobDAO.update_job_status(self.job_id, JobStatusType.PREPARING) def exec(self) -> None: - """job execution""" + """ + Executes the job. + + Parameters + ---------- + + Returns + ------- + NONE + """ + update_job_status(self.job_id, JobStatusType.EXECUTING) + command_list = [self.exec_profile, self.inputs] + print(command_list) def eval(self) -> None: - "job evaluation" + """ + Evaluates the job. + + Parameters + ---------- + + Returns + ------- + NONE + """ + # job_dao = JobDAO() + # job_dao.update_job_status(self.job_id, JobStatusType.EVALUATING) def finalize(self) -> None: - """job finalization""" + """ + Finalizes the job. + + Parameters + ---------- + + Returns + ------- + NONE + """ + # job_dao = JobDAO() + # job_dao.update_job_status(self.job_id, JobStatusType.FINALZING) def cancel(self) -> None: - """job canceling""" + """ + Cancels the job. + + Parameters + ---------- + + Returns + ------- + NONE + """ + # job_dao = JobDAO() + # job_dao.update_job_status(self.job_id, JobStatusType.CANCELED) From 56af3aa2742b8a1a093c66fe64180f0c3f15ae12 Mon Sep 17 00:00:00 2001 From: e-buerger Date: Thu, 5 May 2022 10:00:02 +0200 Subject: [PATCH 06/33] change type of db columns --- exec_manager/dao/db_models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/exec_manager/dao/db_models.py b/exec_manager/dao/db_models.py index a79a7ac..bff4091 100644 --- a/exec_manager/dao/db_models.py +++ b/exec_manager/dao/db_models.py @@ -28,8 +28,8 @@ class DBJob(Base): """An job object stored in the DB""" __tablename__ = "job" - job_id = Column(Integer, primary_key=True) + job_id = Column(String, primary_key=True) job_status = Column(String, nullable=False) - exec_profile = Column(ExecProfile, nullable=False) + exec_profile = Column(JSON, nullable=False) workflow = Column(JSON, nullable=False) - inputs = Column(dict, nullable=False) + inputs = Column(JSON, nullable=False) From 566dbe1ef0b4f13e4612f301cc21b6dfffbb327a Mon Sep 17 00:00:00 2001 From: e-buerger Date: Thu, 5 May 2022 10:02:24 +0200 Subject: [PATCH 07/33] satisfy pre-commit --- exec_manager/dao/db_models.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/exec_manager/dao/db_models.py b/exec_manager/dao/db_models.py index bff4091..5a2ac88 100644 --- a/exec_manager/dao/db_models.py +++ b/exec_manager/dao/db_models.py @@ -15,12 +15,10 @@ """Defines all database specific ORM models""" -from sqlalchemy import JSON, Column, Integer, String +from sqlalchemy import JSON, Column, String from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm.decl_api import DeclarativeMeta -from exec_manager.exec_profile import ExecProfile - Base: DeclarativeMeta = declarative_base() From 37ae4a41f28472fdcd295379b018093f48e1a704 Mon Sep 17 00:00:00 2001 From: e-buerger Date: Mon, 9 May 2022 15:24:28 +0200 Subject: [PATCH 08/33] implement exec method of PythonJob --- exec_manager/dao/__init__.py | 0 exec_manager/dao/db_models.py | 1 + exec_manager/dao/job_dao.py | 35 +++++++++++++------------------ exec_manager/exec_profile.py | 11 ++++++---- exec_manager/job.py | 4 ++-- exec_manager/job_factory.py | 15 +++++++------- exec_manager/python_job.py | 39 ++++++++++++++++++++++++++++------- 7 files changed, 64 insertions(+), 41 deletions(-) create mode 100644 exec_manager/dao/__init__.py diff --git a/exec_manager/dao/__init__.py b/exec_manager/dao/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/exec_manager/dao/db_models.py b/exec_manager/dao/db_models.py index 5a2ac88..56151dd 100644 --- a/exec_manager/dao/db_models.py +++ b/exec_manager/dao/db_models.py @@ -15,6 +15,7 @@ """Defines all database specific ORM models""" + from sqlalchemy import JSON, Column, String from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm.decl_api import DeclarativeMeta diff --git a/exec_manager/dao/job_dao.py b/exec_manager/dao/job_dao.py index a44f96a..4e85ecb 100644 --- a/exec_manager/dao/job_dao.py +++ b/exec_manager/dao/job_dao.py @@ -2,13 +2,14 @@ from uuid import UUID, uuid4 -from sqlalchemy import engine_from_config +from db_models import DBJob +from exec_profile import ExecProfile +from job import Job +from job_status_type import JobStatusType +from sqlalchemy import create_engine from sqlalchemy.orm import Session -from exec_manager.dao.db_models import DBJob -from exec_manager.exec_profile import ExecProfile -from exec_manager.job import Job -from exec_manager.job_status_type import JobStatusType +engine = create_engine("sqlite+pysqlite://") class JobDAO: @@ -41,9 +42,6 @@ class for job dao generates a uuid as job id and checks its uniqueness """ - def __init__(self) -> None: - """constructor""" - def create_job_dao( job_status: JobStatusType, @@ -71,11 +69,8 @@ def create_job_dao( """ job_id = generate_job_id() - with Session(engine_from_config) as session: - with session.begin(): - session.insert(DBJob).values( - job_id, job_status, exec_profile, workflow, inputs - ) + with Session(engine) as session: + session.insert(DBJob).values(job_id, job_status, exec_profile, workflow, inputs) return job_id @@ -111,11 +106,10 @@ def update_job_status(job_id: UUID, new_job_status: JobStatusType) -> None: ------- None """ - with Session(engine_from_config) as session: - with session.begin(): - session.update(DBJob).where(DBJob.job_id == job_id).values( - job_status=new_job_status - ) + with Session(engine) as session: + session.update(DBJob).where(DBJob.job_id == job_id).values( + job_status=new_job_status + ) def get_job(job_id: UUID) -> Job: @@ -131,6 +125,5 @@ def get_job(job_id: UUID) -> Job: ------- Job """ - with Session(engine_from_config) as session: - with session.begin(): - return session.query(DBJob).filter_by(job_id=job_id).all() + with Session(engine) as session: + return session.select(DBJob).where(DBJob.job_id == job_id) diff --git a/exec_manager/exec_profile.py b/exec_manager/exec_profile.py index 66669ae..1f23370 100644 --- a/exec_manager/exec_profile.py +++ b/exec_manager/exec_profile.py @@ -1,7 +1,8 @@ """class for exec profile""" -from exec_manager.exec_profile_type import ExecProfileType -from exec_manager.wf_lang_type import WfLangType +from exec_profile_type import ExecProfileType +from sqlalchemy import JSON +from wf_lang_type import WfLangType class ExecProfile: @@ -21,7 +22,9 @@ class for Exec-Profile ------- """ - def __init__(self, exec_profile_type: ExecProfileType, wf_lang: WfLangType) -> None: + def __init__( + self, exec_profile_type: ExecProfileType, wf_lang: WfLangType, workflow: JSON + ) -> None: """ Constructs all the necessary attributes for the exec profile object. @@ -32,6 +35,6 @@ def __init__(self, exec_profile_type: ExecProfileType, wf_lang: WfLangType) -> N wf_lang : WfLangType workflow language (cwl, wdl, nextflow, snakemake """ - self.exec_profile_type = exec_profile_type self.wf_lang = wf_lang + self.workflow = workflow diff --git a/exec_manager/job.py b/exec_manager/job.py index c958df3..d996d47 100644 --- a/exec_manager/job.py +++ b/exec_manager/job.py @@ -2,8 +2,8 @@ from uuid import UUID -from exec_manager.exec_profile import ExecProfile -from exec_manager.job_status_type import JobStatusType +from exec_profile import ExecProfile +from job_status_type import JobStatusType class Job: diff --git a/exec_manager/job_factory.py b/exec_manager/job_factory.py index 23b9372..47ce81c 100644 --- a/exec_manager/job_factory.py +++ b/exec_manager/job_factory.py @@ -1,11 +1,11 @@ """class for job factory""" -from exec_manager.dao.job_dao import create_job_dao -from exec_manager.exec_profile import ExecProfile -from exec_manager.exec_profile_type import ExecProfileType -from exec_manager.job import Job -from exec_manager.job_status_type import JobStatusType -from exec_manager.python_job import PythonJob +from dao.job_dao import JobDAO +from exec_profile import ExecProfile +from exec_profile_type import ExecProfileType +from job import Job +from job_status_type import JobStatusType +from python_job import PythonJob class JobFactory: @@ -39,7 +39,8 @@ def create_job(inputs: dict, workflow, exec_profile: ExecProfile) -> Job: Job """ job_status = JobStatusType.NOTSTARTET - job_id = create_job_dao(job_status, exec_profile, workflow, inputs) + job_dao = JobDAO() + job_id = job_dao.create_job_dao(job_status, exec_profile, workflow, inputs) if exec_profile.exec_profile_type == ExecProfileType.PYTHON: return PythonJob(job_id, job_status, exec_profile, inputs) if exec_profile.exec_profile_type == ExecProfileType.BASH: diff --git a/exec_manager/python_job.py b/exec_manager/python_job.py index 08c7540..48066e5 100644 --- a/exec_manager/python_job.py +++ b/exec_manager/python_job.py @@ -1,11 +1,15 @@ """class for python job""" +from subprocess import Popen from uuid import UUID -from exec_manager.dao.job_dao import update_job_status -from exec_manager.exec_profile import ExecProfile -from exec_manager.job import Job -from exec_manager.job_status_type import JobStatusType +import yaml +from dao.job_dao import update_job_status +from exec_profile import ExecProfile +from job import Job +from job_status_type import JobStatusType + +from exec_manager.wf_lang_type import WfLangType class PythonJob(Job): @@ -58,7 +62,8 @@ def prepare(self) -> None: ------- NONE """ - # JobDAO.update_job_status(self.job_id, JobStatusType.PREPARING) + # job_dao = JobDAO() + # job_dao.update_job_status(self.job_id, JobStatusType.PREPARING) def exec(self) -> None: """ @@ -72,8 +77,27 @@ def exec(self) -> None: NONE """ update_job_status(self.job_id, JobStatusType.EXECUTING) - command_list = [self.exec_profile, self.inputs] - print(command_list) + if self.exec_profile.wf_lang == WfLangType.CWL: + data = yaml.dump(self.inputs) + with open("inputs.yaml", "w", encoding="utf-8") as input_file: + input_file.write(data) + command_list = ["cwltool ", self.exec_profile.workflow, input_file] + # this line triggers an bandit warning: "Issue:[B603:subprocess_without_shell_equals_ + # true] subprocess call - check for execution of untrusted input." + # This warning is triggered because Popen takes an argument. If this is not an issue, + # you should remove the warning by #nosec after the line + with Popen(command_list) as command_execution: # nosec + exit_code = command_execution.wait() + if exit_code == 0: + update_job_status(JobStatusType.SUCCEEDED) + else: + update_job_status(JobStatusType.FAILED) + elif self.exec_profile.wf_lang == WfLangType.WDL: + pass # insert commands for executing wdl workflows + elif self.exec_profile.wf_lang == WfLangType.SNAKEMAKE: + pass # insert commands for executing snakemake workflows + elif self.exec_profile.wf_lang == WfLangType.NEXTFLOW: + pass # insert commands for executing nextflow workflows def eval(self) -> None: """ @@ -115,4 +139,5 @@ def cancel(self) -> None: NONE """ # job_dao = JobDAO() + # # access Popen object (p) and run p.terminate() # job_dao.update_job_status(self.job_id, JobStatusType.CANCELED) From 55298274115ce088d20ac063c7501fff26807188 Mon Sep 17 00:00:00 2001 From: e-buerger Date: Mon, 9 May 2022 15:34:43 +0200 Subject: [PATCH 09/33] suppres bandit warnigns and add comments --- exec_manager/job_factory.py | 5 ++--- exec_manager/python_job.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/exec_manager/job_factory.py b/exec_manager/job_factory.py index 47ce81c..d5371d0 100644 --- a/exec_manager/job_factory.py +++ b/exec_manager/job_factory.py @@ -1,6 +1,6 @@ """class for job factory""" -from dao.job_dao import JobDAO +from dao.job_dao import create_job_dao from exec_profile import ExecProfile from exec_profile_type import ExecProfileType from job import Job @@ -39,8 +39,7 @@ def create_job(inputs: dict, workflow, exec_profile: ExecProfile) -> Job: Job """ job_status = JobStatusType.NOTSTARTET - job_dao = JobDAO() - job_id = job_dao.create_job_dao(job_status, exec_profile, workflow, inputs) + job_id = create_job_dao(job_status, exec_profile, workflow, inputs) if exec_profile.exec_profile_type == ExecProfileType.PYTHON: return PythonJob(job_id, job_status, exec_profile, inputs) if exec_profile.exec_profile_type == ExecProfileType.BASH: diff --git a/exec_manager/python_job.py b/exec_manager/python_job.py index 48066e5..878d9a6 100644 --- a/exec_manager/python_job.py +++ b/exec_manager/python_job.py @@ -1,6 +1,6 @@ """class for python job""" -from subprocess import Popen +from subprocess import Popen # nosec from uuid import UUID import yaml From 5f17ab34b3fc402c87ebcae2f772294f2f2e36e5 Mon Sep 17 00:00:00 2001 From: e-buerger Date: Mon, 9 May 2022 15:46:32 +0200 Subject: [PATCH 10/33] remove __init__.py from dao directory --- exec_manager/dao/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 exec_manager/dao/__init__.py diff --git a/exec_manager/dao/__init__.py b/exec_manager/dao/__init__.py deleted file mode 100644 index e69de29..0000000 From cd6f4266ad80d213b0d3b269c5d3b014074fb9f1 Mon Sep 17 00:00:00 2001 From: e-buerger Date: Mon, 9 May 2022 15:50:50 +0200 Subject: [PATCH 11/33] remove context manager --- exec_manager/dao/job_dao.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/exec_manager/dao/job_dao.py b/exec_manager/dao/job_dao.py index 4e85ecb..426a4b1 100644 --- a/exec_manager/dao/job_dao.py +++ b/exec_manager/dao/job_dao.py @@ -69,8 +69,8 @@ def create_job_dao( """ job_id = generate_job_id() - with Session(engine) as session: - session.insert(DBJob).values(job_id, job_status, exec_profile, workflow, inputs) + session = Session(engine) + session.insert(DBJob).values(job_id, job_status, exec_profile, workflow, inputs) return job_id @@ -106,10 +106,10 @@ def update_job_status(job_id: UUID, new_job_status: JobStatusType) -> None: ------- None """ - with Session(engine) as session: - session.update(DBJob).where(DBJob.job_id == job_id).values( - job_status=new_job_status - ) + session = Session(engine) + session.update(DBJob).where(DBJob.job_id == job_id).values( + job_status=new_job_status + ) def get_job(job_id: UUID) -> Job: @@ -125,5 +125,5 @@ def get_job(job_id: UUID) -> Job: ------- Job """ - with Session(engine) as session: - return session.select(DBJob).where(DBJob.job_id == job_id) + session = Session(engine) + return session.select(DBJob).where(DBJob.job_id == job_id) From 792ce119c577bd6435bc93566045b0f143e6b5d1 Mon Sep 17 00:00:00 2001 From: e-buerger Date: Mon, 9 May 2022 16:06:48 +0200 Subject: [PATCH 12/33] fix session error in job_dao.py --- exec_manager/dao/job_dao.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/exec_manager/dao/job_dao.py b/exec_manager/dao/job_dao.py index 426a4b1..b4641ba 100644 --- a/exec_manager/dao/job_dao.py +++ b/exec_manager/dao/job_dao.py @@ -6,8 +6,7 @@ from exec_profile import ExecProfile from job import Job from job_status_type import JobStatusType -from sqlalchemy import create_engine -from sqlalchemy.orm import Session +from sqlalchemy import create_engine, insert, select, update engine = create_engine("sqlite+pysqlite://") @@ -67,10 +66,9 @@ def create_job_dao( ------- UUID """ - job_id = generate_job_id() - session = Session(engine) - session.insert(DBJob).values(job_id, job_status, exec_profile, workflow, inputs) + with engine.connect(): + insert(DBJob).values(job_id, job_status, exec_profile, workflow, inputs) return job_id @@ -106,10 +104,8 @@ def update_job_status(job_id: UUID, new_job_status: JobStatusType) -> None: ------- None """ - session = Session(engine) - session.update(DBJob).where(DBJob.job_id == job_id).values( - job_status=new_job_status - ) + with engine.connect(): + update(DBJob).where(DBJob.job_id == job_id).values(job_status=new_job_status) def get_job(job_id: UUID) -> Job: @@ -125,5 +121,5 @@ def get_job(job_id: UUID) -> Job: ------- Job """ - session = Session(engine) - return session.select(DBJob).where(DBJob.job_id == job_id) + with engine.connect(): + return select(DBJob).where(DBJob.job_id == job_id) From 4eeafcf8b705aba7f8e11581cf8e05bee773911c Mon Sep 17 00:00:00 2001 From: e-buerger Date: Mon, 9 May 2022 16:18:15 +0200 Subject: [PATCH 13/33] change db commands --- exec_manager/dao/job_dao.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/exec_manager/dao/job_dao.py b/exec_manager/dao/job_dao.py index b4641ba..5140a75 100644 --- a/exec_manager/dao/job_dao.py +++ b/exec_manager/dao/job_dao.py @@ -67,8 +67,10 @@ def create_job_dao( UUID """ job_id = generate_job_id() - with engine.connect(): - insert(DBJob).values(job_id, job_status, exec_profile, workflow, inputs) + with engine.connect() as connection: + connection.execute( + insert(DBJob).values(job_id, job_status, exec_profile, workflow, inputs) + ) return job_id @@ -104,8 +106,12 @@ def update_job_status(job_id: UUID, new_job_status: JobStatusType) -> None: ------- None """ - with engine.connect(): - update(DBJob).where(DBJob.job_id == job_id).values(job_status=new_job_status) + with engine.connect() as connection: + connection.execute( + update(DBJob) + .where(DBJob.job_id == job_id) + .values(job_status=new_job_status) + ) def get_job(job_id: UUID) -> Job: @@ -121,5 +127,5 @@ def get_job(job_id: UUID) -> Job: ------- Job """ - with engine.connect(): - return select(DBJob).where(DBJob.job_id == job_id) + with engine.connect() as connection: + return connection.execute(select(DBJob).where(DBJob.job_id == job_id)) From af7d7a9ddb324a7010cc24eb7d267c5bbf90a03f Mon Sep 17 00:00:00 2001 From: e-buerger Date: Mon, 9 May 2022 16:20:59 +0200 Subject: [PATCH 14/33] change db commands --- exec_manager/dao/job_dao.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/exec_manager/dao/job_dao.py b/exec_manager/dao/job_dao.py index 5140a75..458dbdf 100644 --- a/exec_manager/dao/job_dao.py +++ b/exec_manager/dao/job_dao.py @@ -69,7 +69,7 @@ def create_job_dao( job_id = generate_job_id() with engine.connect() as connection: connection.execute( - insert(DBJob).values(job_id, job_status, exec_profile, workflow, inputs) + insert("job").values(job_id, job_status, exec_profile, workflow, inputs) ) return job_id @@ -108,7 +108,7 @@ def update_job_status(job_id: UUID, new_job_status: JobStatusType) -> None: """ with engine.connect() as connection: connection.execute( - update(DBJob) + update("job") .where(DBJob.job_id == job_id) .values(job_status=new_job_status) ) @@ -128,4 +128,4 @@ def get_job(job_id: UUID) -> Job: Job """ with engine.connect() as connection: - return connection.execute(select(DBJob).where(DBJob.job_id == job_id)) + return connection.execute(select("job").where(DBJob.job_id == job_id)) From a1d9e618a777bcfa906531cc05dd9d5f7a6767d8 Mon Sep 17 00:00:00 2001 From: e-buerger Date: Mon, 9 May 2022 16:27:51 +0200 Subject: [PATCH 15/33] change db commands --- exec_manager/dao/job_dao.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/exec_manager/dao/job_dao.py b/exec_manager/dao/job_dao.py index 458dbdf..5471a07 100644 --- a/exec_manager/dao/job_dao.py +++ b/exec_manager/dao/job_dao.py @@ -69,7 +69,9 @@ def create_job_dao( job_id = generate_job_id() with engine.connect() as connection: connection.execute( - insert("job").values(job_id, job_status, exec_profile, workflow, inputs) + insert(DBJob.__tablename__).values( + job_id, job_status, exec_profile, workflow, inputs + ) ) return job_id @@ -108,7 +110,7 @@ def update_job_status(job_id: UUID, new_job_status: JobStatusType) -> None: """ with engine.connect() as connection: connection.execute( - update("job") + update(DBJob.__tablename__) .where(DBJob.job_id == job_id) .values(job_status=new_job_status) ) @@ -128,4 +130,4 @@ def get_job(job_id: UUID) -> Job: Job """ with engine.connect() as connection: - return connection.execute(select("job").where(DBJob.job_id == job_id)) + return connection.execute(select().where(DBJob.job_id == job_id)) From 1993d54b1673767271fe15992be21e5f5b25057e Mon Sep 17 00:00:00 2001 From: e-buerger Date: Tue, 10 May 2022 14:06:09 +0200 Subject: [PATCH 16/33] fix import errors by using absolute path and an __init__ file --- exec_manager/dao/__init__.py | 0 exec_manager/dao/job_dao.py | 60 +++++++++++++++++++----------------- exec_manager/exec_profile.py | 7 +++-- exec_manager/job.py | 4 +-- exec_manager/job_factory.py | 23 +++++++++----- exec_manager/python_job.py | 37 ++++++++++++++++------ 6 files changed, 82 insertions(+), 49 deletions(-) create mode 100644 exec_manager/dao/__init__.py diff --git a/exec_manager/dao/__init__.py b/exec_manager/dao/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/exec_manager/dao/job_dao.py b/exec_manager/dao/job_dao.py index 5471a07..af73e8f 100644 --- a/exec_manager/dao/job_dao.py +++ b/exec_manager/dao/job_dao.py @@ -2,44 +2,48 @@ from uuid import UUID, uuid4 -from db_models import DBJob -from exec_profile import ExecProfile -from job import Job -from job_status_type import JobStatusType from sqlalchemy import create_engine, insert, select, update +from exec_manager.dao.db_models import DBJob +from exec_manager.exec_profile import ExecProfile +from exec_manager.job import Job +from exec_manager.job_status_type import JobStatusType + engine = create_engine("sqlite+pysqlite://") -class JobDAO: - """ - class for job dao +# class JobDAO: +# """ +# class for job dao - ... +# ... - Attributes - ---------- +# Attributes +# ---------- - Methods - ------- - create_job( - self, - job_status: JobStatusType, - inputs: dict, - workflow, - exec_profile: ExecProfile, - ) -> UUID: - creates a job +# Methods +# ------- +# create_job( +# self, +# job_status: JobStatusType, +# inputs: dict, +# workflow, +# exec_profile: ExecProfile, +# ) -> UUID: +# creates a job - update_job_status(self, job_id: UUID, new_job_status: JobStatusType) -> None: - updates the status of the job in database +# update_job_status(self, job_id: UUID, new_job_status: JobStatusType) -> None: +# updates the status of the job in database - get_job(job_id: UUID) -> Job: - returns a job by the job id +# get_job(job_id: UUID) -> Job: +# returns a job by the job id - generate_job_id() -> UUID: - generates a uuid as job id and checks its uniqueness - """ +# generate_job_id() -> UUID: +# generates a uuid as job id and checks its uniqueness +# """ + +# def __init__(self) -> None: +# """constructor""" def create_job_dao( @@ -78,7 +82,7 @@ def create_job_dao( def generate_job_id() -> UUID: """ - Generates an unique job id. + Generates a unique job id. Parameters ---------- diff --git a/exec_manager/exec_profile.py b/exec_manager/exec_profile.py index 1f23370..42ce50b 100644 --- a/exec_manager/exec_profile.py +++ b/exec_manager/exec_profile.py @@ -1,8 +1,9 @@ """class for exec profile""" -from exec_profile_type import ExecProfileType from sqlalchemy import JSON -from wf_lang_type import WfLangType + +from exec_manager.exec_profile_type import ExecProfileType +from exec_manager.wf_lang_type import WfLangType class ExecProfile: @@ -34,6 +35,8 @@ def __init__( type of the exec profile (bash, python, wes) wf_lang : WfLangType workflow language (cwl, wdl, nextflow, snakemake + workflow + workflow (file) """ self.exec_profile_type = exec_profile_type self.wf_lang = wf_lang diff --git a/exec_manager/job.py b/exec_manager/job.py index d996d47..c958df3 100644 --- a/exec_manager/job.py +++ b/exec_manager/job.py @@ -2,8 +2,8 @@ from uuid import UUID -from exec_profile import ExecProfile -from job_status_type import JobStatusType +from exec_manager.exec_profile import ExecProfile +from exec_manager.job_status_type import JobStatusType class Job: diff --git a/exec_manager/job_factory.py b/exec_manager/job_factory.py index d5371d0..b3ad8b7 100644 --- a/exec_manager/job_factory.py +++ b/exec_manager/job_factory.py @@ -1,11 +1,13 @@ """class for job factory""" -from dao.job_dao import create_job_dao -from exec_profile import ExecProfile -from exec_profile_type import ExecProfileType -from job import Job -from job_status_type import JobStatusType -from python_job import PythonJob +from typing import Callable + +from exec_manager.dao.job_dao import create_job_dao +from exec_manager.exec_profile import ExecProfile +from exec_manager.exec_profile_type import ExecProfileType +from exec_manager.job import Job +from exec_manager.job_status_type import JobStatusType +from exec_manager.python_job import PythonJob class JobFactory: @@ -27,7 +29,12 @@ def __init__(self) -> None: """this is the constructor""" -def create_job(inputs: dict, workflow, exec_profile: ExecProfile) -> Job: +def create_job( + inputs: dict, + workflow, + exec_profile: ExecProfile, + create_dao_job: Callable = create_job_dao, +) -> Job: """ Creates a job. @@ -39,7 +46,7 @@ def create_job(inputs: dict, workflow, exec_profile: ExecProfile) -> Job: Job """ job_status = JobStatusType.NOTSTARTET - job_id = create_job_dao(job_status, exec_profile, workflow, inputs) + job_id = create_dao_job(job_status, exec_profile, workflow, inputs) if exec_profile.exec_profile_type == ExecProfileType.PYTHON: return PythonJob(job_id, job_status, exec_profile, inputs) if exec_profile.exec_profile_type == ExecProfileType.BASH: diff --git a/exec_manager/python_job.py b/exec_manager/python_job.py index 878d9a6..823deb2 100644 --- a/exec_manager/python_job.py +++ b/exec_manager/python_job.py @@ -4,11 +4,11 @@ from uuid import UUID import yaml -from dao.job_dao import update_job_status -from exec_profile import ExecProfile -from job import Job -from job_status_type import JobStatusType +from exec_manager.dao.job_dao import update_job_status +from exec_manager.exec_profile import ExecProfile +from exec_manager.job import Job +from exec_manager.job_status_type import JobStatusType from exec_manager.wf_lang_type import WfLangType @@ -20,12 +20,14 @@ class for python job Attributes ---------- - job_id : uuid + job_id : UUID id of the job job_status : JobStatusType current status of the job (eg. notstarted, succeeded, failed) exec_profile : ExecProfile python exec profile + inputs : dict + input parameters of the job Methods ------- @@ -38,6 +40,7 @@ class for python job finalize() -> None: finalizes the job cancel() -> None: + cancels the job """ def __init__( @@ -47,7 +50,20 @@ def __init__( exec_profile: ExecProfile, inputs: dict, ) -> None: - """Constructs all the necessary attributes for the python job object.""" + """ + Constructs all the necessary attributes for the python job object. + + Parameters + ---------- + job_id : uuid + id of the job + job_status : JobStatusType + current status of the job (eg. notstarted, succeeded, failed) + exec_profile : ExecProfile + python exec profile + inputs : dict + input parameters of the job + """ Job.__init__(self, job_id, job_status, exec_profile) self.inputs = inputs @@ -62,6 +78,7 @@ def prepare(self) -> None: ------- NONE """ + # setup # job_dao = JobDAO() # job_dao.update_job_status(self.job_id, JobStatusType.PREPARING) @@ -89,9 +106,9 @@ def exec(self) -> None: with Popen(command_list) as command_execution: # nosec exit_code = command_execution.wait() if exit_code == 0: - update_job_status(JobStatusType.SUCCEEDED) + update_job_status(self.job_id, JobStatusType.SUCCEEDED) else: - update_job_status(JobStatusType.FAILED) + update_job_status(self.job_id, JobStatusType.FAILED) elif self.exec_profile.wf_lang == WfLangType.WDL: pass # insert commands for executing wdl workflows elif self.exec_profile.wf_lang == WfLangType.SNAKEMAKE: @@ -110,6 +127,7 @@ def eval(self) -> None: ------- NONE """ + # success or fail # job_dao = JobDAO() # job_dao.update_job_status(self.job_id, JobStatusType.EVALUATING) @@ -124,6 +142,7 @@ def finalize(self) -> None: ------- NONE """ + # teer down # job_dao = JobDAO() # job_dao.update_job_status(self.job_id, JobStatusType.FINALZING) @@ -139,5 +158,5 @@ def cancel(self) -> None: NONE """ # job_dao = JobDAO() - # # access Popen object (p) and run p.terminate() + # # access Popen object (execution) and run execution.terminate() # job_dao.update_job_status(self.job_id, JobStatusType.CANCELED) From ae53782694a26fe86e569f79f57cd9aca85242ff Mon Sep 17 00:00:00 2001 From: e-buerger Date: Tue, 10 May 2022 14:09:48 +0200 Subject: [PATCH 17/33] remove content from exec method --- exec_manager/python_job.py | 52 ++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/exec_manager/python_job.py b/exec_manager/python_job.py index 823deb2..434e09e 100644 --- a/exec_manager/python_job.py +++ b/exec_manager/python_job.py @@ -1,15 +1,17 @@ """class for python job""" -from subprocess import Popen # nosec +# from subprocess import Popen # nosec from uuid import UUID -import yaml - from exec_manager.dao.job_dao import update_job_status from exec_manager.exec_profile import ExecProfile from exec_manager.job import Job from exec_manager.job_status_type import JobStatusType -from exec_manager.wf_lang_type import WfLangType + +# import yaml + + +# from exec_manager.wf_lang_type import WfLangType class PythonJob(Job): @@ -94,27 +96,27 @@ def exec(self) -> None: NONE """ update_job_status(self.job_id, JobStatusType.EXECUTING) - if self.exec_profile.wf_lang == WfLangType.CWL: - data = yaml.dump(self.inputs) - with open("inputs.yaml", "w", encoding="utf-8") as input_file: - input_file.write(data) - command_list = ["cwltool ", self.exec_profile.workflow, input_file] - # this line triggers an bandit warning: "Issue:[B603:subprocess_without_shell_equals_ - # true] subprocess call - check for execution of untrusted input." - # This warning is triggered because Popen takes an argument. If this is not an issue, - # you should remove the warning by #nosec after the line - with Popen(command_list) as command_execution: # nosec - exit_code = command_execution.wait() - if exit_code == 0: - update_job_status(self.job_id, JobStatusType.SUCCEEDED) - else: - update_job_status(self.job_id, JobStatusType.FAILED) - elif self.exec_profile.wf_lang == WfLangType.WDL: - pass # insert commands for executing wdl workflows - elif self.exec_profile.wf_lang == WfLangType.SNAKEMAKE: - pass # insert commands for executing snakemake workflows - elif self.exec_profile.wf_lang == WfLangType.NEXTFLOW: - pass # insert commands for executing nextflow workflows + # if self.exec_profile.wf_lang == WfLangType.CWL: + # data = yaml.dump(self.inputs) + # with open("inputs.yaml", "w", encoding="utf-8") as input_file: + # input_file.write(data) + # command_list = ["cwltool ", self.exec_profile.workflow, input_file] + # # this line triggers an bandit warning: "Issue:[B603:subprocess_without_shell_equals_ + # # true] subprocess call - check for execution of untrusted input." + # # This warning is triggered because Popen takes an argument. If this is not an issue, + # # you should remove the warning by #nosec after the line + # with Popen(command_list) as command_execution: # nosec + # exit_code = command_execution.wait() + # if exit_code == 0: + # update_job_status(self.job_id, JobStatusType.SUCCEEDED) + # else: + # update_job_status(self.job_id, JobStatusType.FAILED) + # elif self.exec_profile.wf_lang == WfLangType.WDL: + # pass # insert commands for executing wdl workflows + # elif self.exec_profile.wf_lang == WfLangType.SNAKEMAKE: + # pass # insert commands for executing snakemake workflows + # elif self.exec_profile.wf_lang == WfLangType.NEXTFLOW: + # pass # insert commands for executing nextflow workflows def eval(self) -> None: """ From 211f348fd58936b8e508d80b205b115da30ba786 Mon Sep 17 00:00:00 2001 From: e-buerger Date: Tue, 10 May 2022 14:13:47 +0200 Subject: [PATCH 18/33] add license headers --- exec_manager/dao/job_dao.py | 15 +++++++++++++++ exec_manager/exec_profile.py | 15 +++++++++++++++ exec_manager/exec_profile_type.py | 15 +++++++++++++++ exec_manager/job.py | 15 +++++++++++++++ exec_manager/job_factory.py | 15 +++++++++++++++ exec_manager/job_status_type.py | 15 +++++++++++++++ exec_manager/python_job.py | 15 +++++++++++++++ exec_manager/wf_lang_type.py | 15 +++++++++++++++ 8 files changed, 120 insertions(+) diff --git a/exec_manager/dao/job_dao.py b/exec_manager/dao/job_dao.py index af73e8f..7b4f6be 100644 --- a/exec_manager/dao/job_dao.py +++ b/exec_manager/dao/job_dao.py @@ -1,3 +1,18 @@ +# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL +# for the German Human Genome-Phenome Archive (GHGA) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """class for job dao""" from uuid import UUID, uuid4 diff --git a/exec_manager/exec_profile.py b/exec_manager/exec_profile.py index 42ce50b..e447831 100644 --- a/exec_manager/exec_profile.py +++ b/exec_manager/exec_profile.py @@ -1,3 +1,18 @@ +# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL +# for the German Human Genome-Phenome Archive (GHGA) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """class for exec profile""" from sqlalchemy import JSON diff --git a/exec_manager/exec_profile_type.py b/exec_manager/exec_profile_type.py index 165d43f..5e7637a 100644 --- a/exec_manager/exec_profile_type.py +++ b/exec_manager/exec_profile_type.py @@ -1,3 +1,18 @@ +# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL +# for the German Human Genome-Phenome Archive (GHGA) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """enum for exec profile type""" from enum import Enum diff --git a/exec_manager/job.py b/exec_manager/job.py index c958df3..61891c9 100644 --- a/exec_manager/job.py +++ b/exec_manager/job.py @@ -1,3 +1,18 @@ +# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL +# for the German Human Genome-Phenome Archive (GHGA) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """class for job""" from uuid import UUID diff --git a/exec_manager/job_factory.py b/exec_manager/job_factory.py index b3ad8b7..9d1dda6 100644 --- a/exec_manager/job_factory.py +++ b/exec_manager/job_factory.py @@ -1,3 +1,18 @@ +# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL +# for the German Human Genome-Phenome Archive (GHGA) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """class for job factory""" from typing import Callable diff --git a/exec_manager/job_status_type.py b/exec_manager/job_status_type.py index 2b33bfc..26e7b5b 100644 --- a/exec_manager/job_status_type.py +++ b/exec_manager/job_status_type.py @@ -1,3 +1,18 @@ +# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL +# for the German Human Genome-Phenome Archive (GHGA) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """enum for job status type""" from enum import Enum diff --git a/exec_manager/python_job.py b/exec_manager/python_job.py index 434e09e..fda5ca9 100644 --- a/exec_manager/python_job.py +++ b/exec_manager/python_job.py @@ -1,3 +1,18 @@ +# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL +# for the German Human Genome-Phenome Archive (GHGA) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """class for python job""" # from subprocess import Popen # nosec diff --git a/exec_manager/wf_lang_type.py b/exec_manager/wf_lang_type.py index 25cb214..7fd85f6 100644 --- a/exec_manager/wf_lang_type.py +++ b/exec_manager/wf_lang_type.py @@ -1,3 +1,18 @@ +# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL +# for the German Human Genome-Phenome Archive (GHGA) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """enum for workflow languagae type""" from enum import Enum From 46f313eff4b625196848863393a435e9c3504ff8 Mon Sep 17 00:00:00 2001 From: e-buerger Date: Tue, 10 May 2022 14:16:17 +0200 Subject: [PATCH 19/33] add license headers --- exec_manager/dao/__init__.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/exec_manager/dao/__init__.py b/exec_manager/dao/__init__.py index e69de29..19e3303 100644 --- a/exec_manager/dao/__init__.py +++ b/exec_manager/dao/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL +# for the German Human Genome-Phenome Archive (GHGA) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. From c5253c31c0fbca3208ab09085336779a1d0c7acc Mon Sep 17 00:00:00 2001 From: e-buerger Date: Fri, 13 May 2022 11:05:33 +0200 Subject: [PATCH 20/33] implement PyExecSession --- exec_manager/dao/db_models.py | 2 + exec_manager/dao/job_dao.py | 42 +++++++++++----- exec_manager/exec_profile.py | 7 +-- exec_manager/job.py | 67 +++++++++++++++++++++++-- exec_manager/job_status_type.py | 2 +- exec_manager/py_exec_session.py | 88 +++++++++++++++++++++++++++++++++ exec_manager/python_job.py | 43 ++-------------- 7 files changed, 188 insertions(+), 63 deletions(-) create mode 100644 exec_manager/py_exec_session.py diff --git a/exec_manager/dao/db_models.py b/exec_manager/dao/db_models.py index 56151dd..6c3ab62 100644 --- a/exec_manager/dao/db_models.py +++ b/exec_manager/dao/db_models.py @@ -21,12 +21,14 @@ from sqlalchemy.orm.decl_api import DeclarativeMeta Base: DeclarativeMeta = declarative_base() +metadata = Base.metadata class DBJob(Base): """An job object stored in the DB""" __tablename__ = "job" + job_id = Column(String, primary_key=True) job_status = Column(String, nullable=False) exec_profile = Column(JSON, nullable=False) diff --git a/exec_manager/dao/job_dao.py b/exec_manager/dao/job_dao.py index 7b4f6be..24151ca 100644 --- a/exec_manager/dao/job_dao.py +++ b/exec_manager/dao/job_dao.py @@ -15,17 +15,17 @@ """class for job dao""" +import json from uuid import UUID, uuid4 -from sqlalchemy import create_engine, insert, select, update +from sqlalchemy import JSON, create_engine, insert, select, update from exec_manager.dao.db_models import DBJob from exec_manager.exec_profile import ExecProfile +from exec_manager.exec_profile_type import ExecProfileType from exec_manager.job import Job from exec_manager.job_status_type import JobStatusType - -engine = create_engine("sqlite+pysqlite://") - +from exec_manager.wf_lang_type import WfLangType # class JobDAO: # """ @@ -60,11 +60,14 @@ # def __init__(self) -> None: # """constructor""" +# global engine +engine = create_engine("sqlite+pysqlite://") + def create_job_dao( job_status: JobStatusType, exec_profile: ExecProfile, - workflow, + workflow: JSON, inputs: dict, ) -> UUID: """ @@ -86,10 +89,19 @@ def create_job_dao( UUID """ job_id = generate_job_id() + job_id_str = str(job_id) + job_status_str = job_status.value + exec_profile_json = json.dumps( + { + "exec_profile_type": exec_profile.exec_profile_type.value, + "wf_lang": exec_profile.wf_lang.value, + } + ) + inputs_json = json.dumps(inputs) with engine.connect() as connection: connection.execute( - insert(DBJob.__tablename__).values( - job_id, job_status, exec_profile, workflow, inputs + insert(DBJob).values( + (job_id_str, job_status_str, exec_profile_json, workflow, inputs_json) ) ) return job_id @@ -107,8 +119,8 @@ def generate_job_id() -> UUID: UUID """ job_id = uuid4() - while get_job(job_id) is not None: - job_id = uuid4() + # while get_job(job_id, engine) is not None: + # job_id = uuid4() return job_id @@ -129,7 +141,7 @@ def update_job_status(job_id: UUID, new_job_status: JobStatusType) -> None: """ with engine.connect() as connection: connection.execute( - update(DBJob.__tablename__) + update(DBJob) .where(DBJob.job_id == job_id) .values(job_status=new_job_status) ) @@ -149,4 +161,12 @@ def get_job(job_id: UUID) -> Job: Job """ with engine.connect() as connection: - return connection.execute(select().where(DBJob.job_id == job_id)) + cursor = connection.execute(select(DBJob).where(DBJob.job_id == str(job_id))) + result = cursor.fetchall() + job_status = JobStatusType(result[0][1]) + exec_profile = json.loads(result[0][2]) + exec_profile = ExecProfile( + ExecProfileType(exec_profile["exec_profile_type"]), + WfLangType(exec_profile["wf_lang"]), + ) + return Job(job_id, job_status, exec_profile) diff --git a/exec_manager/exec_profile.py b/exec_manager/exec_profile.py index e447831..e0913b1 100644 --- a/exec_manager/exec_profile.py +++ b/exec_manager/exec_profile.py @@ -15,8 +15,6 @@ """class for exec profile""" -from sqlalchemy import JSON - from exec_manager.exec_profile_type import ExecProfileType from exec_manager.wf_lang_type import WfLangType @@ -38,9 +36,7 @@ class for Exec-Profile ------- """ - def __init__( - self, exec_profile_type: ExecProfileType, wf_lang: WfLangType, workflow: JSON - ) -> None: + def __init__(self, exec_profile_type: ExecProfileType, wf_lang: WfLangType) -> None: """ Constructs all the necessary attributes for the exec profile object. @@ -55,4 +51,3 @@ def __init__( """ self.exec_profile_type = exec_profile_type self.wf_lang = wf_lang - self.workflow = workflow diff --git a/exec_manager/job.py b/exec_manager/job.py index 61891c9..d679639 100644 --- a/exec_manager/job.py +++ b/exec_manager/job.py @@ -69,17 +69,74 @@ def __init__( self.job_status = job_status self.exec_profile = exec_profile + def __eq__(self, __o: object) -> bool: + if isinstance(__o, Job): + return ( + self.job_id == __o.job_id + and self.job_status == __o.job_status + and self.exec_profile == __o.exec_profile + ) + return NotImplemented + + # def __hash__(self) -> int: + # return hash((self.job_id, self.job_status, self.exec_profile)) + def prepare(self) -> None: - """This method should implement the prepare execution step""" + """ + Prepares the job. + + Parameters + ---------- + + Returns + ------- + NONE + """ def exec(self) -> None: - """This method should implement the exec execution step""" + """ + Executes the job. + + Parameters + ---------- + + Returns + ------- + NONE + """ def eval(self) -> None: - """This method should implement the eval execution step""" + """ + Evaluates the job. + + Parameters + ---------- + + Returns + ------- + NONE + """ def finalize(self) -> None: - """This method should implement the finalize execution step""" + """ + Finalizes the job. + + Parameters + ---------- + + Returns + ------- + NONE + """ def cancel(self) -> None: - """This method should implement the cancel execution step""" + """ + Cancels the job. + + Parameters + ---------- + + Returns + ------- + NONE + """ diff --git a/exec_manager/job_status_type.py b/exec_manager/job_status_type.py index 26e7b5b..afbd87e 100644 --- a/exec_manager/job_status_type.py +++ b/exec_manager/job_status_type.py @@ -21,7 +21,7 @@ class JobStatusType(Enum): """enumerate job status types""" - NOTSTARTET = "not startetd" + NOTSTARTET = "not started" """job ist not started yet""" PREPARING = "preparing" diff --git a/exec_manager/py_exec_session.py b/exec_manager/py_exec_session.py new file mode 100644 index 0000000..56cd961 --- /dev/null +++ b/exec_manager/py_exec_session.py @@ -0,0 +1,88 @@ +# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL +# for the German Human Genome-Phenome Archive (GHGA) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""class for python exec session""" + + +from uuid import UUID + +from job_dao import get_job, update_job_status +from sqlalchemy import Integer + +from exec_manager.exec_profile import ExecProfile +from exec_manager.job_status_type import JobStatusType +from exec_manager.python_job import PythonJob + + +class PyExecSession: + """ + class for python job + + ... + + Attributes + ---------- + max_retries : int + + Methods + ------- + start() -> None + starts a session + """ + + def __init__( + self, + max_retries: Integer = 0, + ) -> None: + """ + Constructs all the necessary attributes for the python exec session. + + Parameters + ---------- + max_retries : int + """ + self.max_retries = max_retries + + def run( + self, + job_id: UUID, + job_status: JobStatusType, + exec_profile: ExecProfile, + inputs: dict, + ) -> None: + """ + Starts the session. + + Parameters + ---------- + + Returns + ------- + NONE + """ + counter = -1 + while self.max_retries > counter: + python_job = PythonJob(job_id, job_status, exec_profile, inputs) + update_job_status(job_id, JobStatusType.PREPARING) + python_job.prepare() + update_job_status(job_id, JobStatusType.EXECUTING) + python_job.exec() + update_job_status(job_id, JobStatusType.EVALUATING) + python_job.eval() + update_job_status(job_id, JobStatusType.FINALZING) + python_job.finalize() + if get_job(job_id).job_status == JobStatusType.SUCCEEDED: + break + counter = counter + 1 diff --git a/exec_manager/python_job.py b/exec_manager/python_job.py index fda5ca9..421eeb4 100644 --- a/exec_manager/python_job.py +++ b/exec_manager/python_job.py @@ -18,16 +18,10 @@ # from subprocess import Popen # nosec from uuid import UUID -from exec_manager.dao.job_dao import update_job_status from exec_manager.exec_profile import ExecProfile from exec_manager.job import Job from exec_manager.job_status_type import JobStatusType -# import yaml - - -# from exec_manager.wf_lang_type import WfLangType - class PythonJob(Job): """ @@ -80,9 +74,12 @@ def __init__( python exec profile inputs : dict input parameters of the job + py_exec_session : PyExecSession + session in which the job will be run """ Job.__init__(self, job_id, job_status, exec_profile) self.inputs = inputs + self.wf_lang = exec_profile.wf_lang def prepare(self) -> None: """ @@ -95,9 +92,6 @@ def prepare(self) -> None: ------- NONE """ - # setup - # job_dao = JobDAO() - # job_dao.update_job_status(self.job_id, JobStatusType.PREPARING) def exec(self) -> None: """ @@ -110,28 +104,6 @@ def exec(self) -> None: ------- NONE """ - update_job_status(self.job_id, JobStatusType.EXECUTING) - # if self.exec_profile.wf_lang == WfLangType.CWL: - # data = yaml.dump(self.inputs) - # with open("inputs.yaml", "w", encoding="utf-8") as input_file: - # input_file.write(data) - # command_list = ["cwltool ", self.exec_profile.workflow, input_file] - # # this line triggers an bandit warning: "Issue:[B603:subprocess_without_shell_equals_ - # # true] subprocess call - check for execution of untrusted input." - # # This warning is triggered because Popen takes an argument. If this is not an issue, - # # you should remove the warning by #nosec after the line - # with Popen(command_list) as command_execution: # nosec - # exit_code = command_execution.wait() - # if exit_code == 0: - # update_job_status(self.job_id, JobStatusType.SUCCEEDED) - # else: - # update_job_status(self.job_id, JobStatusType.FAILED) - # elif self.exec_profile.wf_lang == WfLangType.WDL: - # pass # insert commands for executing wdl workflows - # elif self.exec_profile.wf_lang == WfLangType.SNAKEMAKE: - # pass # insert commands for executing snakemake workflows - # elif self.exec_profile.wf_lang == WfLangType.NEXTFLOW: - # pass # insert commands for executing nextflow workflows def eval(self) -> None: """ @@ -144,9 +116,6 @@ def eval(self) -> None: ------- NONE """ - # success or fail - # job_dao = JobDAO() - # job_dao.update_job_status(self.job_id, JobStatusType.EVALUATING) def finalize(self) -> None: """ @@ -159,9 +128,6 @@ def finalize(self) -> None: ------- NONE """ - # teer down - # job_dao = JobDAO() - # job_dao.update_job_status(self.job_id, JobStatusType.FINALZING) def cancel(self) -> None: """ @@ -174,6 +140,3 @@ def cancel(self) -> None: ------- NONE """ - # job_dao = JobDAO() - # # access Popen object (execution) and run execution.terminate() - # job_dao.update_job_status(self.job_id, JobStatusType.CANCELED) From 97c902a6c79a177e96878650f55db7f018ccd0e9 Mon Sep 17 00:00:00 2001 From: e-buerger Date: Mon, 16 May 2022 11:45:56 +0200 Subject: [PATCH 21/33] implement unit tests for job_dao --- exec_manager/dao/job_dao.py | 62 +++++++++-------- exec_manager/job.py | 9 --- exec_manager/py_exec_session.py | 6 +- tests/unit/test_job_dao.py | 120 ++++++++++++++++++++++++++++++++ 4 files changed, 155 insertions(+), 42 deletions(-) create mode 100644 tests/unit/test_job_dao.py diff --git a/exec_manager/dao/job_dao.py b/exec_manager/dao/job_dao.py index 24151ca..1ef8884 100644 --- a/exec_manager/dao/job_dao.py +++ b/exec_manager/dao/job_dao.py @@ -20,7 +20,7 @@ from sqlalchemy import JSON, create_engine, insert, select, update -from exec_manager.dao.db_models import DBJob +from exec_manager.dao.db_models import DBJob, metadata from exec_manager.exec_profile import ExecProfile from exec_manager.exec_profile_type import ExecProfileType from exec_manager.job import Job @@ -60,8 +60,8 @@ # def __init__(self) -> None: # """constructor""" -# global engine -engine = create_engine("sqlite+pysqlite://") +DB_ENGINE = create_engine("sqlite+pysqlite://") +metadata.create_all(DB_ENGINE) def create_job_dao( @@ -98,7 +98,7 @@ def create_job_dao( } ) inputs_json = json.dumps(inputs) - with engine.connect() as connection: + with DB_ENGINE.connect() as connection: connection.execute( insert(DBJob).values( (job_id_str, job_status_str, exec_profile_json, workflow, inputs_json) @@ -107,21 +107,33 @@ def create_job_dao( return job_id -def generate_job_id() -> UUID: +def get_job(job_id: UUID) -> Job: """ - Generates a unique job id. + Returns a job by his job id. Parameters ---------- + job_id: UUID + id of the job Returns ------- - UUID + Job """ - job_id = uuid4() - # while get_job(job_id, engine) is not None: - # job_id = uuid4() - return job_id + with DB_ENGINE.connect() as connection: + cursor = connection.execute( + select([DBJob.job_id, DBJob.job_status, DBJob.exec_profile]).where( + DBJob.job_id == str(job_id) + ) + ) + result = cursor.fetchall() + job_status = JobStatusType(result[0][1]) + exec_profile = json.loads(result[0][2]) + exec_profile = ExecProfile( + ExecProfileType(exec_profile["exec_profile_type"]), + WfLangType(exec_profile["wf_lang"]), + ) + return Job(job_id, job_status, exec_profile) def update_job_status(job_id: UUID, new_job_status: JobStatusType) -> None: @@ -139,34 +151,26 @@ def update_job_status(job_id: UUID, new_job_status: JobStatusType) -> None: ------- None """ - with engine.connect() as connection: + with DB_ENGINE.connect() as connection: connection.execute( update(DBJob) - .where(DBJob.job_id == job_id) - .values(job_status=new_job_status) + .where(DBJob.job_id == str(job_id)) + .values(job_status=new_job_status.value) ) -def get_job(job_id: UUID) -> Job: +def generate_job_id() -> UUID: """ - Returns a job by his job id. + Generates a unique job id. Parameters ---------- - job_id: UUID - id of the job Returns ------- - Job + UUID """ - with engine.connect() as connection: - cursor = connection.execute(select(DBJob).where(DBJob.job_id == str(job_id))) - result = cursor.fetchall() - job_status = JobStatusType(result[0][1]) - exec_profile = json.loads(result[0][2]) - exec_profile = ExecProfile( - ExecProfileType(exec_profile["exec_profile_type"]), - WfLangType(exec_profile["wf_lang"]), - ) - return Job(job_id, job_status, exec_profile) + job_id = uuid4() + # while get_job(job_id, engine) is not None: + # job_id = uuid4() + return job_id diff --git a/exec_manager/job.py b/exec_manager/job.py index d679639..ab5fb8c 100644 --- a/exec_manager/job.py +++ b/exec_manager/job.py @@ -69,15 +69,6 @@ def __init__( self.job_status = job_status self.exec_profile = exec_profile - def __eq__(self, __o: object) -> bool: - if isinstance(__o, Job): - return ( - self.job_id == __o.job_id - and self.job_status == __o.job_status - and self.exec_profile == __o.exec_profile - ) - return NotImplemented - # def __hash__(self) -> int: # return hash((self.job_id, self.job_status, self.exec_profile)) diff --git a/exec_manager/py_exec_session.py b/exec_manager/py_exec_session.py index 56cd961..460ad72 100644 --- a/exec_manager/py_exec_session.py +++ b/exec_manager/py_exec_session.py @@ -18,9 +18,7 @@ from uuid import UUID -from job_dao import get_job, update_job_status -from sqlalchemy import Integer - +from exec_manager.dao.job_dao import get_job, update_job_status from exec_manager.exec_profile import ExecProfile from exec_manager.job_status_type import JobStatusType from exec_manager.python_job import PythonJob @@ -44,7 +42,7 @@ class for python job def __init__( self, - max_retries: Integer = 0, + max_retries: int = 0, ) -> None: """ Constructs all the necessary attributes for the python exec session. diff --git a/tests/unit/test_job_dao.py b/tests/unit/test_job_dao.py new file mode 100644 index 0000000..eaf2ed4 --- /dev/null +++ b/tests/unit/test_job_dao.py @@ -0,0 +1,120 @@ +import json + +from exec_manager.dao.job_dao import create_job_dao, get_job, update_job_status +from exec_manager.exec_profile import ExecProfile +from exec_manager.exec_profile_type import ExecProfileType +from exec_manager.job_status_type import JobStatusType +from exec_manager.wf_lang_type import WfLangType + +# from uuid import UUID + + +# @pytest.fixture +# def example_engine(): +# engine = create_engine("sqlite+pysqlite://") +# metadata.create_all(engine) +# return engine + + +# @pytest.fixture +# def example_job_id(): +# return create_job_dao( +# JobStatusType.NOTSTARTET, +# ExecProfile(ExecProfileType.PYTHON, WfLangType.CWL), +# json.dumps({"test": 1}), +# {"hello": "world"}, +# ) + + +# engine = create_engine("sqlite+pysqlite://") +# metadata.create_all(engine) +job_status = JobStatusType.NOTSTARTET +exec_profile = ExecProfile(ExecProfileType.PYTHON, WfLangType.CWL) +workflow = "test_workflow.json" # {"test" : 1} +inputs = {"hello": "world"} +example_job_id = create_job_dao( + JobStatusType.NOTSTARTET, + ExecProfile(ExecProfileType.PYTHON, WfLangType.CWL), + json.dumps({"test": 1}), + {"hello": "world"}, +) + + +def test_create_job_dao(): + job_status = JobStatusType.NOTSTARTET + exec_profile = ExecProfile(ExecProfileType.PYTHON, WfLangType.CWL) + workflow = json.dumps({"test": 1}) + inputs = {"hello": "world"} + job_id = create_job_dao( + job_status, + exec_profile, + workflow, + inputs, + ) + db_job = get_job(job_id) + assert ( + str(db_job.job_id) == str(job_id) + and db_job.job_status.value == job_status.value + and ( + json.dumps( + { + "exec_profile_type": db_job.exec_profile.exec_profile_type.value, + "wf_lang": db_job.exec_profile.wf_lang.value, + } + ) + == json.dumps( + { + "exec_profile_type": ExecProfileType.PYTHON.value, + "wf_lang": WfLangType.CWL.value, + } + ) + ) + ) + + +# convert every object to string to check if the content of the objects are equal +def test_get_job(): + job_id = example_job_id + db_job = get_job(job_id) + assert ( + str(db_job.job_id) == str(job_id) + and db_job.job_status.value == JobStatusType.NOTSTARTET.value + and ( + json.dumps( + { + "exec_profile_type": db_job.exec_profile.exec_profile_type.value, + "wf_lang": db_job.exec_profile.wf_lang.value, + } + ) + == json.dumps( + { + "exec_profile_type": ExecProfileType.PYTHON.value, + "wf_lang": WfLangType.CWL.value, + } + ) + ) + ) + + +def test_update_job_status(): + job_id = example_job_id + update_job_status(job_id, JobStatusType.NOTSTARTET) + db_job = get_job(job_id) + assert ( + str(db_job.job_id) == str(job_id) + and db_job.job_status.value == JobStatusType.NOTSTARTET.value + and ( + json.dumps( + { + "exec_profile_type": db_job.exec_profile.exec_profile_type.value, + "wf_lang": db_job.exec_profile.wf_lang.value, + } + ) + == json.dumps( + { + "exec_profile_type": ExecProfileType.PYTHON.value, + "wf_lang": WfLangType.CWL.value, + } + ) + ) + ) From 6491949c36155a6f6cb68be409360fa10124fae5 Mon Sep 17 00:00:00 2001 From: e-buerger Date: Mon, 16 May 2022 12:13:06 +0200 Subject: [PATCH 22/33] satisfy pipeline --- exec_manager/dao/job_dao.py | 10 +++++----- tests/unit/test_job_dao.py | 16 ++++++---------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/exec_manager/dao/job_dao.py b/exec_manager/dao/job_dao.py index 1ef8884..63d968d 100644 --- a/exec_manager/dao/job_dao.py +++ b/exec_manager/dao/job_dao.py @@ -18,7 +18,7 @@ import json from uuid import UUID, uuid4 -from sqlalchemy import JSON, create_engine, insert, select, update +from sqlalchemy import create_engine, insert, select, update from exec_manager.dao.db_models import DBJob, metadata from exec_manager.exec_profile import ExecProfile @@ -67,7 +67,7 @@ def create_job_dao( job_status: JobStatusType, exec_profile: ExecProfile, - workflow: JSON, + workflow: dict, inputs: dict, ) -> UUID: """ @@ -100,7 +100,7 @@ def create_job_dao( inputs_json = json.dumps(inputs) with DB_ENGINE.connect() as connection: connection.execute( - insert(DBJob).values( + insert(DBJob.__table__).values( (job_id_str, job_status_str, exec_profile_json, workflow, inputs_json) ) ) @@ -122,7 +122,7 @@ def get_job(job_id: UUID) -> Job: """ with DB_ENGINE.connect() as connection: cursor = connection.execute( - select([DBJob.job_id, DBJob.job_status, DBJob.exec_profile]).where( + select(DBJob.job_id, DBJob.job_status, DBJob.exec_profile).where( DBJob.job_id == str(job_id) ) ) @@ -153,7 +153,7 @@ def update_job_status(job_id: UUID, new_job_status: JobStatusType) -> None: """ with DB_ENGINE.connect() as connection: connection.execute( - update(DBJob) + update(DBJob.__table__) .where(DBJob.job_id == str(job_id)) .values(job_status=new_job_status.value) ) diff --git a/tests/unit/test_job_dao.py b/tests/unit/test_job_dao.py index eaf2ed4..bfb82f4 100644 --- a/tests/unit/test_job_dao.py +++ b/tests/unit/test_job_dao.py @@ -30,21 +30,17 @@ # metadata.create_all(engine) job_status = JobStatusType.NOTSTARTET exec_profile = ExecProfile(ExecProfileType.PYTHON, WfLangType.CWL) -workflow = "test_workflow.json" # {"test" : 1} +workflow = {"test": 1} inputs = {"hello": "world"} example_job_id = create_job_dao( JobStatusType.NOTSTARTET, ExecProfile(ExecProfileType.PYTHON, WfLangType.CWL), - json.dumps({"test": 1}), - {"hello": "world"}, + workflow, + inputs, ) def test_create_job_dao(): - job_status = JobStatusType.NOTSTARTET - exec_profile = ExecProfile(ExecProfileType.PYTHON, WfLangType.CWL) - workflow = json.dumps({"test": 1}) - inputs = {"hello": "world"} job_id = create_job_dao( job_status, exec_profile, @@ -78,7 +74,7 @@ def test_get_job(): db_job = get_job(job_id) assert ( str(db_job.job_id) == str(job_id) - and db_job.job_status.value == JobStatusType.NOTSTARTET.value + and db_job.job_status.value == job_status.value and ( json.dumps( { @@ -98,11 +94,11 @@ def test_get_job(): def test_update_job_status(): job_id = example_job_id - update_job_status(job_id, JobStatusType.NOTSTARTET) + update_job_status(job_id, JobStatusType.PREPARING) db_job = get_job(job_id) assert ( str(db_job.job_id) == str(job_id) - and db_job.job_status.value == JobStatusType.NOTSTARTET.value + and db_job.job_status.value == JobStatusType.PREPARING.value and ( json.dumps( { From c3449999eafa7c8b7586a7e12a8c692acd58d18a Mon Sep 17 00:00:00 2001 From: e-buerger Date: Mon, 16 May 2022 12:15:34 +0200 Subject: [PATCH 23/33] satisfy pipeline --- exec_manager/dao/job_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exec_manager/dao/job_dao.py b/exec_manager/dao/job_dao.py index 63d968d..50d2d6a 100644 --- a/exec_manager/dao/job_dao.py +++ b/exec_manager/dao/job_dao.py @@ -122,7 +122,7 @@ def get_job(job_id: UUID) -> Job: """ with DB_ENGINE.connect() as connection: cursor = connection.execute( - select(DBJob.job_id, DBJob.job_status, DBJob.exec_profile).where( + select([DBJob.job_id, DBJob.job_status, DBJob.exec_profile]).where( DBJob.job_id == str(job_id) ) ) From d93859cc2c9da1f94648c3e801a8318f89611e19 Mon Sep 17 00:00:00 2001 From: e-buerger Date: Mon, 16 May 2022 12:18:37 +0200 Subject: [PATCH 24/33] add license header --- tests/unit/test_job_dao.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/unit/test_job_dao.py b/tests/unit/test_job_dao.py index bfb82f4..888a2a8 100644 --- a/tests/unit/test_job_dao.py +++ b/tests/unit/test_job_dao.py @@ -1,3 +1,18 @@ +# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL +# for the German Human Genome-Phenome Archive (GHGA) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import json from exec_manager.dao.job_dao import create_job_dao, get_job, update_job_status From c862e18d7d62fed4a999b7fb38338c1f5a64802e Mon Sep 17 00:00:00 2001 From: e-buerger Date: Tue, 17 May 2022 13:27:58 +0200 Subject: [PATCH 25/33] update README --- README.md | 164 ++++++++++-------------------------------------------- 1 file changed, 28 insertions(+), 136 deletions(-) diff --git a/README.md b/README.md index 836f544..198c806 100644 --- a/README.md +++ b/README.md @@ -1,143 +1,35 @@ -# Microservice Repository Template - -This repo is a template for creating a new microservice. - -The directories, files, and their structure herein are recommendations -from the GHGA Dev Team. - -## Naming Conventions -The github repository contains only lowercase letters, numbers, and hyphens "-", -e.g.: `my-microservice` - -The python package (and thus the source repository) contains underscores "_" -instead of hyphens, e.g.: `exec_manager` - -The command-line script that is used to run the service, the docker repository -(published to docker hub), and the helm chart (not part of this repository) use the -same pattern as the repository name, e.g.: `my-microservice` -## Adapt to your service -This is just a template and needs some adaption to your specific use case. - -Please search for **"please adapt"** comments. They will indicate all locations -that need modification. Once the adaptions are in place, please remove these # -comments. - -The following should serve as a template for the final repo's README, -please adapt it accordingly (e.g. replace all occurences of `my-microservice` or `exec_manager` with the final package name and don't forget to adapt the links): - ---- - -**\# please adapt the links of following badges:** -![tests](https://github.com/ghga-de/my-microservice/actions/workflows/unit_and_int_tests.yaml/badge.svg) -[![Coverage Status](https://coveralls.io/repos/github/ghga-de/my-microservice/badge.svg?branch=main)](https://coveralls.io/github/ghga-de/my-microservice?branch=main) -# My-Microservice - -A description explaining the use case of this service. - -## Documentation: - -An extensive documentation can be found [here](...) (coming soon). - -## Quick Start -### Installation -We recommend using the provided Docker container. - -A pre-build version is available at [docker hub](https://hub.docker.com/repository/docker/ghga/my-microservice): -```bash -# Please feel free to choose the version as needed: -docker pull ghga/my-microservice: -``` - -Or you can build the container yourself from the [`./Dockerfile`](./Dockerfile): -```bash -# Execute in the repo's root dir: -# (Please feel free to adapt the name/tag.) -docker build -t ghga/my-microservice: . -``` - -For production-ready deployment, we recommend using Kubernetes, however, -for simple use cases, you could execute the service using docker -on a single server: -```bash -# The entrypoint is preconfigured: -docker run -p 8080:8080 ghga/my-microservice: -``` - -If you prefer not to use containers, you may install the service from source: -```bash -# Execute in the repo's root dir: -pip install . - -# to run the service: -my-microservice -``` - -### Configuration: -The [`./example-config.yaml`](./example-config.yaml) gives an overview of the available configuration options. -Please adapt it and choose one of the following options for injecting it into the service: -- specify the path to via the `exec_manager_CONFIG_YAML` env variable -- rename it to `.exec_manager.yaml` and place it into one of the following locations: - - the current working directory were you are execute the service (on unix: `./.exec_manager.yaml`) - - your home directory (on unix: `~/.exec_manager.yaml`) - -The config yaml will be automatically parsed by the service. - -**Important: If you are using containers, the locations refer to paths within the container.** - -All parameters mentioned in the [`./example-config.yaml`](./example-config.yaml) -could also be set using environment variables or file secrets. - -For naming the environment variables, just prefix the parameter name with `exec_manager_`, -e.g. for the `host` set an environment variable named `exec_manager_HOST` -(you may use both upper or lower cases, however, it is standard to define all env -variables in upper cases). - -To using file secrets please refer to the -[corresponding section](https://pydantic-docs.helpmanual.io/usage/settings/#secret-support) -of the pydantic documentation. - - -## Development -For setting up the development environment, we rely on the -[devcontainer feature](https://code.visualstudio.com/docs/remote/containers) of vscode -in combination with Docker Compose. - -To use it, you have to have Docker Compose as well as vscode with its "Remote - Containers" extension (`ms-vscode-remote.remote-containers`) installed. -Then open this repository in vscode and run the command -`Remote-Containers: Reopen in Container` from the vscode "Command Palette". - -This will give you a full-fledged, pre-configured development environment including: -- infrastructural dependencies of the service (databases, etc.) -- all relevant vscode extensions pre-installed -- pre-configured linting and auto-formating -- a pre-configured debugger -- automatic license-header insertion - -Moreover, inside the devcontainer, there are two convenience commands available -(please type them in the integrated terminal of vscode): -- `dev_install` - install the service with all development dependencies, -installs pre-commit, and applies any migration scripts to the test database -(please run that if you are starting the devcontainer for the first time -or if you added any python dependencies to the [`./setup.cfg`](./setup.cfg)) -- `dev_launcher` - starts the service with the development config yaml -(located in the `./.devcontainer/` dir) - -If you prefer not to use vscode, you could get a similar setup (without the editor specific features) -by running the following commands: -``` bash -# Execute in the repo's root dir: -cd ./.devcontainer - -# build and run the environment with docker-compose -docker-compose up - -# attach to the main container: -# (you can open multiple shell sessions like this) -docker exec -it devcontainer_app_1 /bin/bash +# Execution Manager for WorkflUX + +The execution manager manages the execution of jobs which will be runned with workflUX. There will be three types to execute a workflow: by Python, Bash or WES. + +## Execution Profiles +Yet, there is only the python exec profile but in future there will be the bash exec profile and the WES exec profile as well. The execution contains four steps: prepare, exec, eval, finalize. But only the exec step is required and the others are optional. +- __prepare:__ +This step will be executed before the actual workflow execution. For example there can be load required python or conda environments. +- __exec:__ +This step will execute the actual workflow and is the only required step. At the end of this step, the status of the job should be updated depending on the exit code of the job execution. +- __eval:__ +This step can evaluate the success of the workflow execution. But the exit code in the exec step should be used to set the new status (FAILED or SUCCEDED) of the job. + +- __finalize:__ +This step will be executed at the end of the whole job execution. It can be used for cleaning up temporary files. + + +### Python +For the python exec profile you have to implement the exec method from the PythonJob class. Therefore you create a new python file which contains a class that inherit the PythonJob class. Then you implement at least the exec method. +After that you have to create yaml file which looks like the file below: +```yaml +EXEC_PROFILES: + NAMEOFEXECPROFILE: + type: python + max_retries: 2 # please adat this number + py_module: ./python_script_with_implemented_methods.py + py_class: ClassOfImplementedMethods ``` +```max_retries``` gives an numeric value for the maximum retries when the execution (consisting of the four steps) fails. ## License This repository is free to use and modify according to the [Apache 2.0 License](./LICENSE). From 37b711cc76f9909e4843eb0630ac8bb3da3107ca Mon Sep 17 00:00:00 2001 From: e-buerger Date: Tue, 17 May 2022 13:34:02 +0200 Subject: [PATCH 26/33] update doc strings; change job_dao functions for better unit tests --- README.md | 10 +++--- exec_manager/dao/job_dao.py | 58 ++++++++++----------------------- exec_manager/exec_profile.py | 6 ++-- exec_manager/job_factory.py | 19 ----------- exec_manager/py_exec_session.py | 1 + tests/unit/test_job_dao.py | 15 ++++----- 6 files changed, 32 insertions(+), 77 deletions(-) diff --git a/README.md b/README.md index 198c806..e1905c1 100644 --- a/README.md +++ b/README.md @@ -7,14 +7,14 @@ The execution manager manages the execution of jobs which will be runned with wo ## Execution Profiles Yet, there is only the python exec profile but in future there will be the bash exec profile and the WES exec profile as well. The execution contains four steps: prepare, exec, eval, finalize. But only the exec step is required and the others are optional. -- __prepare:__ +- __prepare:__ This step will be executed before the actual workflow execution. For example there can be load required python or conda environments. -- __exec:__ +- __exec:__ This step will execute the actual workflow and is the only required step. At the end of this step, the status of the job should be updated depending on the exit code of the job execution. -- __eval:__ +- __eval:__ This step can evaluate the success of the workflow execution. But the exit code in the exec step should be used to set the new status (FAILED or SUCCEDED) of the job. -- __finalize:__ +- __finalize:__ This step will be executed at the end of the whole job execution. It can be used for cleaning up temporary files. @@ -23,7 +23,7 @@ For the python exec profile you have to implement the exec method from the Pytho After that you have to create yaml file which looks like the file below: ```yaml EXEC_PROFILES: - NAMEOFEXECPROFILE: + NAMEOFEXECPROFILE: type: python max_retries: 2 # please adat this number py_module: ./python_script_with_implemented_methods.py diff --git a/exec_manager/dao/job_dao.py b/exec_manager/dao/job_dao.py index 50d2d6a..b1b7855 100644 --- a/exec_manager/dao/job_dao.py +++ b/exec_manager/dao/job_dao.py @@ -18,7 +18,7 @@ import json from uuid import UUID, uuid4 -from sqlalchemy import create_engine, insert, select, update +from sqlalchemy import create_engine, engine, insert, select, update from exec_manager.dao.db_models import DBJob, metadata from exec_manager.exec_profile import ExecProfile @@ -27,39 +27,6 @@ from exec_manager.job_status_type import JobStatusType from exec_manager.wf_lang_type import WfLangType -# class JobDAO: -# """ -# class for job dao - -# ... - -# Attributes -# ---------- - -# Methods -# ------- -# create_job( -# self, -# job_status: JobStatusType, -# inputs: dict, -# workflow, -# exec_profile: ExecProfile, -# ) -> UUID: -# creates a job - -# update_job_status(self, job_id: UUID, new_job_status: JobStatusType) -> None: -# updates the status of the job in database - -# get_job(job_id: UUID) -> Job: -# returns a job by the job id - -# generate_job_id() -> UUID: -# generates a uuid as job id and checks its uniqueness -# """ - -# def __init__(self) -> None: -# """constructor""" - DB_ENGINE = create_engine("sqlite+pysqlite://") metadata.create_all(DB_ENGINE) @@ -69,6 +36,7 @@ def create_job_dao( exec_profile: ExecProfile, workflow: dict, inputs: dict, + db_engine: engine = DB_ENGINE, ) -> UUID: """ Inserts a job into the database. @@ -83,6 +51,8 @@ def create_job_dao( the jobs workflow inputs: dict the input parameters of the job + engine: engine + db engine where the connection will be established (default is sqlite with pysqlite) Returns ------- @@ -98,7 +68,7 @@ def create_job_dao( } ) inputs_json = json.dumps(inputs) - with DB_ENGINE.connect() as connection: + with db_engine.connect() as connection: connection.execute( insert(DBJob.__table__).values( (job_id_str, job_status_str, exec_profile_json, workflow, inputs_json) @@ -107,7 +77,7 @@ def create_job_dao( return job_id -def get_job(job_id: UUID) -> Job: +def get_job(job_id: UUID, db_engine: engine = DB_ENGINE) -> Job: """ Returns a job by his job id. @@ -115,12 +85,14 @@ def get_job(job_id: UUID) -> Job: ---------- job_id: UUID id of the job + engine: engine + db engine where the connection will be established (default is sqlite with pysqlite) Returns ------- Job """ - with DB_ENGINE.connect() as connection: + with db_engine.connect() as connection: cursor = connection.execute( select([DBJob.job_id, DBJob.job_status, DBJob.exec_profile]).where( DBJob.job_id == str(job_id) @@ -136,7 +108,9 @@ def get_job(job_id: UUID) -> Job: return Job(job_id, job_status, exec_profile) -def update_job_status(job_id: UUID, new_job_status: JobStatusType) -> None: +def update_job_status( + job_id: UUID, new_job_status: JobStatusType, db_engine: engine = DB_ENGINE +) -> None: """ Updates a jobs status by his job id. @@ -146,12 +120,14 @@ def update_job_status(job_id: UUID, new_job_status: JobStatusType) -> None: id of the job new_job_status: JobStatusType new status of the job; cannot be JobStatusType.NOTSTARTED + engine: engine + db engine where the connection will be established (default is sqlite with pysqlite) Returns ------- None """ - with DB_ENGINE.connect() as connection: + with db_engine.connect() as connection: connection.execute( update(DBJob.__table__) .where(DBJob.job_id == str(job_id)) @@ -171,6 +147,6 @@ def generate_job_id() -> UUID: UUID """ job_id = uuid4() - # while get_job(job_id, engine) is not None: - # job_id = uuid4() + # while get_job(job_id) is not None: + # job_id = uuid4() return job_id diff --git a/exec_manager/exec_profile.py b/exec_manager/exec_profile.py index e0913b1..b7a8f63 100644 --- a/exec_manager/exec_profile.py +++ b/exec_manager/exec_profile.py @@ -30,7 +30,7 @@ class for Exec-Profile exec_profile_type : ExecProfileType type of the exec profile (bash, python, wes) wf_lang : WfLangType - workflow language (cwl, wdl, nextflow, snakemake + workflow language (cwl, wdl, nextflow, snakemake) Methods ------- @@ -45,9 +45,7 @@ def __init__(self, exec_profile_type: ExecProfileType, wf_lang: WfLangType) -> N exec_profile_type : ExecProfileType type of the exec profile (bash, python, wes) wf_lang : WfLangType - workflow language (cwl, wdl, nextflow, snakemake - workflow - workflow (file) + workflow language (cwl, wdl, nextflow, snakemake) """ self.exec_profile_type = exec_profile_type self.wf_lang = wf_lang diff --git a/exec_manager/job_factory.py b/exec_manager/job_factory.py index 9d1dda6..b9ffd35 100644 --- a/exec_manager/job_factory.py +++ b/exec_manager/job_factory.py @@ -25,25 +25,6 @@ from exec_manager.python_job import PythonJob -class JobFactory: - """ - class for job factory - - ... - - Attributes - ---------- - - Methods - ------- - create_job(inputs: dict, workflow, exec_profile: ExecProfile) -> Job: - creates a new job - """ - - def __init__(self) -> None: - """this is the constructor""" - - def create_job( inputs: dict, workflow, diff --git a/exec_manager/py_exec_session.py b/exec_manager/py_exec_session.py index 460ad72..bb6e616 100644 --- a/exec_manager/py_exec_session.py +++ b/exec_manager/py_exec_session.py @@ -50,6 +50,7 @@ def __init__( Parameters ---------- max_retries : int + number of maximum retries when the execution fails """ self.max_retries = max_retries diff --git a/tests/unit/test_job_dao.py b/tests/unit/test_job_dao.py index 888a2a8..0e04a5d 100644 --- a/tests/unit/test_job_dao.py +++ b/tests/unit/test_job_dao.py @@ -21,7 +21,9 @@ from exec_manager.job_status_type import JobStatusType from exec_manager.wf_lang_type import WfLangType -# from uuid import UUID +# import pytest +# from exec_manager.dao.db_models import metadata +# from sqlalchemy import create_engine # @pytest.fixture @@ -55,13 +57,9 @@ ) +# @pytest.mark.usefixtures("example_engine") def test_create_job_dao(): - job_id = create_job_dao( - job_status, - exec_profile, - workflow, - inputs, - ) + job_id = create_job_dao(job_status, exec_profile, workflow, inputs) db_job = get_job(job_id) assert ( str(db_job.job_id) == str(job_id) @@ -83,7 +81,7 @@ def test_create_job_dao(): ) -# convert every object to string to check if the content of the objects are equal +# @pytest.mark.usefixtures("example_engine") def test_get_job(): job_id = example_job_id db_job = get_job(job_id) @@ -107,6 +105,7 @@ def test_get_job(): ) +# @pytest.mark.usefixtures("example_engine") def test_update_job_status(): job_id = example_job_id update_job_status(job_id, JobStatusType.PREPARING) From bf7255db41ed88e474fbb87448a922adb65f8809 Mon Sep 17 00:00:00 2001 From: e-buerger Date: Tue, 17 May 2022 13:37:40 +0200 Subject: [PATCH 27/33] change engine type --- exec_manager/dao/job_dao.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/exec_manager/dao/job_dao.py b/exec_manager/dao/job_dao.py index b1b7855..f0f0c7f 100644 --- a/exec_manager/dao/job_dao.py +++ b/exec_manager/dao/job_dao.py @@ -18,7 +18,8 @@ import json from uuid import UUID, uuid4 -from sqlalchemy import create_engine, engine, insert, select, update +from sqlalchemy import create_engine, insert, select, update +from sqlalchemy.engine import Engine from exec_manager.dao.db_models import DBJob, metadata from exec_manager.exec_profile import ExecProfile @@ -36,7 +37,7 @@ def create_job_dao( exec_profile: ExecProfile, workflow: dict, inputs: dict, - db_engine: engine = DB_ENGINE, + db_engine: Engine = DB_ENGINE, ) -> UUID: """ Inserts a job into the database. @@ -77,7 +78,7 @@ def create_job_dao( return job_id -def get_job(job_id: UUID, db_engine: engine = DB_ENGINE) -> Job: +def get_job(job_id: UUID, db_engine: Engine = DB_ENGINE) -> Job: """ Returns a job by his job id. @@ -109,7 +110,7 @@ def get_job(job_id: UUID, db_engine: engine = DB_ENGINE) -> Job: def update_job_status( - job_id: UUID, new_job_status: JobStatusType, db_engine: engine = DB_ENGINE + job_id: UUID, new_job_status: JobStatusType, db_engine: Engine = DB_ENGINE ) -> None: """ Updates a jobs status by his job id. From f671e4388474ef643e178633a95fcf656f43b3bf Mon Sep 17 00:00:00 2001 From: e-buerger Date: Wed, 4 May 2022 09:07:37 +0200 Subject: [PATCH 28/33] draft of OOP structure for python-centric jobs create classfiles adapting code for passing precommit adapt code for passing precommit adapt code for passing precommit implement a few functions change type of db columns satisfy pre-commit implement exec method of PythonJob suppres bandit warnigns and add comments remove __init__.py from dao directory remove context manager fix session error in job_dao.py change db commands change db commands change db commands fix import errors by using absolute path and an __init__ file remove content from exec method add license headers add license headers implement PyExecSession implement unit tests for job_dao satisfy pipeline satisfy pipeline add license header update README update doc strings; change job_dao functions for better unit tests change engine type --- README.md | 164 +++++------------------------- exec_manager/__init__.py | 2 +- exec_manager/dao/__init__.py | 14 +++ exec_manager/dao/db_models.py | 26 ++--- exec_manager/dao/job_dao.py | 153 ++++++++++++++++++++++++++++ exec_manager/exec_profile.py | 51 ++++++++++ exec_manager/exec_profile_type.py | 31 ++++++ exec_manager/job.py | 133 ++++++++++++++++++++++++ exec_manager/job_factory.py | 52 ++++++++++ exec_manager/job_status_type.py | 46 +++++++++ exec_manager/py_exec_session.py | 87 ++++++++++++++++ exec_manager/python_job.py | 142 ++++++++++++++++++++++++++ exec_manager/wf_lang_type.py | 34 +++++++ tests/unit/test_job_dao.py | 130 +++++++++++++++++++++++ 14 files changed, 913 insertions(+), 152 deletions(-) create mode 100644 exec_manager/dao/__init__.py create mode 100644 exec_manager/dao/job_dao.py create mode 100644 exec_manager/exec_profile.py create mode 100644 exec_manager/exec_profile_type.py create mode 100644 exec_manager/job.py create mode 100644 exec_manager/job_factory.py create mode 100644 exec_manager/job_status_type.py create mode 100644 exec_manager/py_exec_session.py create mode 100644 exec_manager/python_job.py create mode 100644 exec_manager/wf_lang_type.py create mode 100644 tests/unit/test_job_dao.py diff --git a/README.md b/README.md index 836f544..e1905c1 100644 --- a/README.md +++ b/README.md @@ -1,143 +1,35 @@ -# Microservice Repository Template - -This repo is a template for creating a new microservice. - -The directories, files, and their structure herein are recommendations -from the GHGA Dev Team. - -## Naming Conventions -The github repository contains only lowercase letters, numbers, and hyphens "-", -e.g.: `my-microservice` - -The python package (and thus the source repository) contains underscores "_" -instead of hyphens, e.g.: `exec_manager` - -The command-line script that is used to run the service, the docker repository -(published to docker hub), and the helm chart (not part of this repository) use the -same pattern as the repository name, e.g.: `my-microservice` -## Adapt to your service -This is just a template and needs some adaption to your specific use case. - -Please search for **"please adapt"** comments. They will indicate all locations -that need modification. Once the adaptions are in place, please remove these # -comments. - -The following should serve as a template for the final repo's README, -please adapt it accordingly (e.g. replace all occurences of `my-microservice` or `exec_manager` with the final package name and don't forget to adapt the links): - ---- - -**\# please adapt the links of following badges:** -![tests](https://github.com/ghga-de/my-microservice/actions/workflows/unit_and_int_tests.yaml/badge.svg) -[![Coverage Status](https://coveralls.io/repos/github/ghga-de/my-microservice/badge.svg?branch=main)](https://coveralls.io/github/ghga-de/my-microservice?branch=main) -# My-Microservice - -A description explaining the use case of this service. - -## Documentation: - -An extensive documentation can be found [here](...) (coming soon). - -## Quick Start -### Installation -We recommend using the provided Docker container. - -A pre-build version is available at [docker hub](https://hub.docker.com/repository/docker/ghga/my-microservice): -```bash -# Please feel free to choose the version as needed: -docker pull ghga/my-microservice: -``` - -Or you can build the container yourself from the [`./Dockerfile`](./Dockerfile): -```bash -# Execute in the repo's root dir: -# (Please feel free to adapt the name/tag.) -docker build -t ghga/my-microservice: . -``` - -For production-ready deployment, we recommend using Kubernetes, however, -for simple use cases, you could execute the service using docker -on a single server: -```bash -# The entrypoint is preconfigured: -docker run -p 8080:8080 ghga/my-microservice: -``` - -If you prefer not to use containers, you may install the service from source: -```bash -# Execute in the repo's root dir: -pip install . - -# to run the service: -my-microservice -``` - -### Configuration: -The [`./example-config.yaml`](./example-config.yaml) gives an overview of the available configuration options. -Please adapt it and choose one of the following options for injecting it into the service: -- specify the path to via the `exec_manager_CONFIG_YAML` env variable -- rename it to `.exec_manager.yaml` and place it into one of the following locations: - - the current working directory were you are execute the service (on unix: `./.exec_manager.yaml`) - - your home directory (on unix: `~/.exec_manager.yaml`) - -The config yaml will be automatically parsed by the service. - -**Important: If you are using containers, the locations refer to paths within the container.** - -All parameters mentioned in the [`./example-config.yaml`](./example-config.yaml) -could also be set using environment variables or file secrets. - -For naming the environment variables, just prefix the parameter name with `exec_manager_`, -e.g. for the `host` set an environment variable named `exec_manager_HOST` -(you may use both upper or lower cases, however, it is standard to define all env -variables in upper cases). - -To using file secrets please refer to the -[corresponding section](https://pydantic-docs.helpmanual.io/usage/settings/#secret-support) -of the pydantic documentation. - - -## Development -For setting up the development environment, we rely on the -[devcontainer feature](https://code.visualstudio.com/docs/remote/containers) of vscode -in combination with Docker Compose. - -To use it, you have to have Docker Compose as well as vscode with its "Remote - Containers" extension (`ms-vscode-remote.remote-containers`) installed. -Then open this repository in vscode and run the command -`Remote-Containers: Reopen in Container` from the vscode "Command Palette". - -This will give you a full-fledged, pre-configured development environment including: -- infrastructural dependencies of the service (databases, etc.) -- all relevant vscode extensions pre-installed -- pre-configured linting and auto-formating -- a pre-configured debugger -- automatic license-header insertion - -Moreover, inside the devcontainer, there are two convenience commands available -(please type them in the integrated terminal of vscode): -- `dev_install` - install the service with all development dependencies, -installs pre-commit, and applies any migration scripts to the test database -(please run that if you are starting the devcontainer for the first time -or if you added any python dependencies to the [`./setup.cfg`](./setup.cfg)) -- `dev_launcher` - starts the service with the development config yaml -(located in the `./.devcontainer/` dir) - -If you prefer not to use vscode, you could get a similar setup (without the editor specific features) -by running the following commands: -``` bash -# Execute in the repo's root dir: -cd ./.devcontainer - -# build and run the environment with docker-compose -docker-compose up - -# attach to the main container: -# (you can open multiple shell sessions like this) -docker exec -it devcontainer_app_1 /bin/bash +# Execution Manager for WorkflUX + +The execution manager manages the execution of jobs which will be runned with workflUX. There will be three types to execute a workflow: by Python, Bash or WES. + +## Execution Profiles +Yet, there is only the python exec profile but in future there will be the bash exec profile and the WES exec profile as well. The execution contains four steps: prepare, exec, eval, finalize. But only the exec step is required and the others are optional. +- __prepare:__ +This step will be executed before the actual workflow execution. For example there can be load required python or conda environments. +- __exec:__ +This step will execute the actual workflow and is the only required step. At the end of this step, the status of the job should be updated depending on the exit code of the job execution. +- __eval:__ +This step can evaluate the success of the workflow execution. But the exit code in the exec step should be used to set the new status (FAILED or SUCCEDED) of the job. + +- __finalize:__ +This step will be executed at the end of the whole job execution. It can be used for cleaning up temporary files. + + +### Python +For the python exec profile you have to implement the exec method from the PythonJob class. Therefore you create a new python file which contains a class that inherit the PythonJob class. Then you implement at least the exec method. +After that you have to create yaml file which looks like the file below: +```yaml +EXEC_PROFILES: + NAMEOFEXECPROFILE: + type: python + max_retries: 2 # please adat this number + py_module: ./python_script_with_implemented_methods.py + py_class: ClassOfImplementedMethods ``` +```max_retries``` gives an numeric value for the maximum retries when the execution (consisting of the four steps) fails. ## License This repository is free to use and modify according to the [Apache 2.0 License](./LICENSE). diff --git a/exec_manager/__init__.py b/exec_manager/__init__.py index 7589079..0921524 100644 --- a/exec_manager/__init__.py +++ b/exec_manager/__init__.py @@ -13,6 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Short description of package""" # Please adapt to package +"""backend""" # Please adapt to package __version__ = "0.1.0" diff --git a/exec_manager/dao/__init__.py b/exec_manager/dao/__init__.py new file mode 100644 index 0000000..19e3303 --- /dev/null +++ b/exec_manager/dao/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL +# for the German Human Genome-Phenome Archive (GHGA) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/exec_manager/dao/db_models.py b/exec_manager/dao/db_models.py index 805180e..6c3ab62 100644 --- a/exec_manager/dao/db_models.py +++ b/exec_manager/dao/db_models.py @@ -15,26 +15,22 @@ """Defines all database specific ORM models""" -from sqlalchemy import JSON, Boolean, Column, Integer, String + +from sqlalchemy import JSON, Column, String from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm.decl_api import DeclarativeMeta Base: DeclarativeMeta = declarative_base() +metadata = Base.metadata -class ExampleObjectA(Base): - """An example object stored in the DB""" - - __tablename__ = "visas" - id = Column(Integer, primary_key=True) - name = Column(String, nullable=False) - some_json_details = Column(JSON, nullable=False) - +class DBJob(Base): + """An job object stored in the DB""" -class ExampleObjectB(Base): - """Another example object stored in the DB""" + __tablename__ = "job" - __tablename__ = "table_b" - id = Column(Integer, primary_key=True) - name = Column(String, nullable=False) - active = Column(Boolean, nullable=False) + job_id = Column(String, primary_key=True) + job_status = Column(String, nullable=False) + exec_profile = Column(JSON, nullable=False) + workflow = Column(JSON, nullable=False) + inputs = Column(JSON, nullable=False) diff --git a/exec_manager/dao/job_dao.py b/exec_manager/dao/job_dao.py new file mode 100644 index 0000000..f0f0c7f --- /dev/null +++ b/exec_manager/dao/job_dao.py @@ -0,0 +1,153 @@ +# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL +# for the German Human Genome-Phenome Archive (GHGA) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""class for job dao""" + +import json +from uuid import UUID, uuid4 + +from sqlalchemy import create_engine, insert, select, update +from sqlalchemy.engine import Engine + +from exec_manager.dao.db_models import DBJob, metadata +from exec_manager.exec_profile import ExecProfile +from exec_manager.exec_profile_type import ExecProfileType +from exec_manager.job import Job +from exec_manager.job_status_type import JobStatusType +from exec_manager.wf_lang_type import WfLangType + +DB_ENGINE = create_engine("sqlite+pysqlite://") +metadata.create_all(DB_ENGINE) + + +def create_job_dao( + job_status: JobStatusType, + exec_profile: ExecProfile, + workflow: dict, + inputs: dict, + db_engine: Engine = DB_ENGINE, +) -> UUID: + """ + Inserts a job into the database. + + Parameters + ---------- + job_status: JobStatusType + current status of the job; initially it is JobStatusType.NOTSTARTED + exec_profile: ExecProfile + exec profile of this job + workflow + the jobs workflow + inputs: dict + the input parameters of the job + engine: engine + db engine where the connection will be established (default is sqlite with pysqlite) + + Returns + ------- + UUID + """ + job_id = generate_job_id() + job_id_str = str(job_id) + job_status_str = job_status.value + exec_profile_json = json.dumps( + { + "exec_profile_type": exec_profile.exec_profile_type.value, + "wf_lang": exec_profile.wf_lang.value, + } + ) + inputs_json = json.dumps(inputs) + with db_engine.connect() as connection: + connection.execute( + insert(DBJob.__table__).values( + (job_id_str, job_status_str, exec_profile_json, workflow, inputs_json) + ) + ) + return job_id + + +def get_job(job_id: UUID, db_engine: Engine = DB_ENGINE) -> Job: + """ + Returns a job by his job id. + + Parameters + ---------- + job_id: UUID + id of the job + engine: engine + db engine where the connection will be established (default is sqlite with pysqlite) + + Returns + ------- + Job + """ + with db_engine.connect() as connection: + cursor = connection.execute( + select([DBJob.job_id, DBJob.job_status, DBJob.exec_profile]).where( + DBJob.job_id == str(job_id) + ) + ) + result = cursor.fetchall() + job_status = JobStatusType(result[0][1]) + exec_profile = json.loads(result[0][2]) + exec_profile = ExecProfile( + ExecProfileType(exec_profile["exec_profile_type"]), + WfLangType(exec_profile["wf_lang"]), + ) + return Job(job_id, job_status, exec_profile) + + +def update_job_status( + job_id: UUID, new_job_status: JobStatusType, db_engine: Engine = DB_ENGINE +) -> None: + """ + Updates a jobs status by his job id. + + Parameters + ---------- + job_id: UUID + id of the job + new_job_status: JobStatusType + new status of the job; cannot be JobStatusType.NOTSTARTED + engine: engine + db engine where the connection will be established (default is sqlite with pysqlite) + + Returns + ------- + None + """ + with db_engine.connect() as connection: + connection.execute( + update(DBJob.__table__) + .where(DBJob.job_id == str(job_id)) + .values(job_status=new_job_status.value) + ) + + +def generate_job_id() -> UUID: + """ + Generates a unique job id. + + Parameters + ---------- + + Returns + ------- + UUID + """ + job_id = uuid4() + # while get_job(job_id) is not None: + # job_id = uuid4() + return job_id diff --git a/exec_manager/exec_profile.py b/exec_manager/exec_profile.py new file mode 100644 index 0000000..b7a8f63 --- /dev/null +++ b/exec_manager/exec_profile.py @@ -0,0 +1,51 @@ +# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL +# for the German Human Genome-Phenome Archive (GHGA) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""class for exec profile""" + +from exec_manager.exec_profile_type import ExecProfileType +from exec_manager.wf_lang_type import WfLangType + + +class ExecProfile: + """ + class for Exec-Profile + + ... + + Attributes + ---------- + exec_profile_type : ExecProfileType + type of the exec profile (bash, python, wes) + wf_lang : WfLangType + workflow language (cwl, wdl, nextflow, snakemake) + + Methods + ------- + """ + + def __init__(self, exec_profile_type: ExecProfileType, wf_lang: WfLangType) -> None: + """ + Constructs all the necessary attributes for the exec profile object. + + Parameters + ---------- + exec_profile_type : ExecProfileType + type of the exec profile (bash, python, wes) + wf_lang : WfLangType + workflow language (cwl, wdl, nextflow, snakemake) + """ + self.exec_profile_type = exec_profile_type + self.wf_lang = wf_lang diff --git a/exec_manager/exec_profile_type.py b/exec_manager/exec_profile_type.py new file mode 100644 index 0000000..5e7637a --- /dev/null +++ b/exec_manager/exec_profile_type.py @@ -0,0 +1,31 @@ +# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL +# for the German Human Genome-Phenome Archive (GHGA) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""enum for exec profile type""" + +from enum import Enum + + +class ExecProfileType(Enum): + """enumaerate exec profile types""" + + BASH = "bash" + """execution with bash""" + + PYTHON = "python" + """execution with python""" + + WES = "wes" + """execution with wes""" diff --git a/exec_manager/job.py b/exec_manager/job.py new file mode 100644 index 0000000..ab5fb8c --- /dev/null +++ b/exec_manager/job.py @@ -0,0 +1,133 @@ +# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL +# for the German Human Genome-Phenome Archive (GHGA) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""class for job""" + +from uuid import UUID + +from exec_manager.exec_profile import ExecProfile +from exec_manager.job_status_type import JobStatusType + + +class Job: + """ + class for job + + ... + + Attributes + ---------- + job_id : uuid + id of the job + job_status : JobStatusType + current status of the job (eg. notstarted, succeeded, failed) + exec_profile : ExecProfile + exec profile with which the job should be executed + + Methods + ------- + prepare() -> None: + prepares the job + exec() -> None: + executes the job + eval() -> None: + evaluates the job + finalize() -> None: + finalizes the job + cancel() -> None: + cancels the job + """ + + def __init__( + self, job_id: UUID, job_status: JobStatusType, exec_profile: ExecProfile + ) -> None: + """ + Constructs all the necessary attributes for the job object. + + Parameters + ---------- + job_id : uuid + id of the job + job_status : JobStatusType + current status of the job (eg. notstarted, succeeded, failed) + exec_profile : ExecProfile + exec profile with which the job should be executed + """ + self.job_id = job_id + self.job_status = job_status + self.exec_profile = exec_profile + + # def __hash__(self) -> int: + # return hash((self.job_id, self.job_status, self.exec_profile)) + + def prepare(self) -> None: + """ + Prepares the job. + + Parameters + ---------- + + Returns + ------- + NONE + """ + + def exec(self) -> None: + """ + Executes the job. + + Parameters + ---------- + + Returns + ------- + NONE + """ + + def eval(self) -> None: + """ + Evaluates the job. + + Parameters + ---------- + + Returns + ------- + NONE + """ + + def finalize(self) -> None: + """ + Finalizes the job. + + Parameters + ---------- + + Returns + ------- + NONE + """ + + def cancel(self) -> None: + """ + Cancels the job. + + Parameters + ---------- + + Returns + ------- + NONE + """ diff --git a/exec_manager/job_factory.py b/exec_manager/job_factory.py new file mode 100644 index 0000000..b9ffd35 --- /dev/null +++ b/exec_manager/job_factory.py @@ -0,0 +1,52 @@ +# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL +# for the German Human Genome-Phenome Archive (GHGA) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""class for job factory""" + +from typing import Callable + +from exec_manager.dao.job_dao import create_job_dao +from exec_manager.exec_profile import ExecProfile +from exec_manager.exec_profile_type import ExecProfileType +from exec_manager.job import Job +from exec_manager.job_status_type import JobStatusType +from exec_manager.python_job import PythonJob + + +def create_job( + inputs: dict, + workflow, + exec_profile: ExecProfile, + create_dao_job: Callable = create_job_dao, +) -> Job: + """ + Creates a job. + + Parameters + ---------- + + Returns + ------- + Job + """ + job_status = JobStatusType.NOTSTARTET + job_id = create_dao_job(job_status, exec_profile, workflow, inputs) + if exec_profile.exec_profile_type == ExecProfileType.PYTHON: + return PythonJob(job_id, job_status, exec_profile, inputs) + if exec_profile.exec_profile_type == ExecProfileType.BASH: + pass # place for bash job + if exec_profile.exec_profile_type == ExecProfileType.WES: + pass # place for wes job + return Job(job_id, job_status, exec_profile) diff --git a/exec_manager/job_status_type.py b/exec_manager/job_status_type.py new file mode 100644 index 0000000..afbd87e --- /dev/null +++ b/exec_manager/job_status_type.py @@ -0,0 +1,46 @@ +# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL +# for the German Human Genome-Phenome Archive (GHGA) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""enum for job status type""" + +from enum import Enum + + +class JobStatusType(Enum): + """enumerate job status types""" + + NOTSTARTET = "not started" + """job ist not started yet""" + + PREPARING = "preparing" + """job is preparing""" + + EXECUTING = "executing" + """job is executing""" + + EVALUATING = "evaluating" + """job ist evaluating""" + + FINALZING = "finalizing" + """job is finalizing""" + + CANCELED = "canceled" + """job is canceled""" + + FAILED = "failed" + """job is failed""" + + SUCCEEDED = "succeeded" + """job is succeeded""" diff --git a/exec_manager/py_exec_session.py b/exec_manager/py_exec_session.py new file mode 100644 index 0000000..bb6e616 --- /dev/null +++ b/exec_manager/py_exec_session.py @@ -0,0 +1,87 @@ +# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL +# for the German Human Genome-Phenome Archive (GHGA) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""class for python exec session""" + + +from uuid import UUID + +from exec_manager.dao.job_dao import get_job, update_job_status +from exec_manager.exec_profile import ExecProfile +from exec_manager.job_status_type import JobStatusType +from exec_manager.python_job import PythonJob + + +class PyExecSession: + """ + class for python job + + ... + + Attributes + ---------- + max_retries : int + + Methods + ------- + start() -> None + starts a session + """ + + def __init__( + self, + max_retries: int = 0, + ) -> None: + """ + Constructs all the necessary attributes for the python exec session. + + Parameters + ---------- + max_retries : int + number of maximum retries when the execution fails + """ + self.max_retries = max_retries + + def run( + self, + job_id: UUID, + job_status: JobStatusType, + exec_profile: ExecProfile, + inputs: dict, + ) -> None: + """ + Starts the session. + + Parameters + ---------- + + Returns + ------- + NONE + """ + counter = -1 + while self.max_retries > counter: + python_job = PythonJob(job_id, job_status, exec_profile, inputs) + update_job_status(job_id, JobStatusType.PREPARING) + python_job.prepare() + update_job_status(job_id, JobStatusType.EXECUTING) + python_job.exec() + update_job_status(job_id, JobStatusType.EVALUATING) + python_job.eval() + update_job_status(job_id, JobStatusType.FINALZING) + python_job.finalize() + if get_job(job_id).job_status == JobStatusType.SUCCEEDED: + break + counter = counter + 1 diff --git a/exec_manager/python_job.py b/exec_manager/python_job.py new file mode 100644 index 0000000..421eeb4 --- /dev/null +++ b/exec_manager/python_job.py @@ -0,0 +1,142 @@ +# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL +# for the German Human Genome-Phenome Archive (GHGA) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""class for python job""" + +# from subprocess import Popen # nosec +from uuid import UUID + +from exec_manager.exec_profile import ExecProfile +from exec_manager.job import Job +from exec_manager.job_status_type import JobStatusType + + +class PythonJob(Job): + """ + class for python job + + ... + + Attributes + ---------- + job_id : UUID + id of the job + job_status : JobStatusType + current status of the job (eg. notstarted, succeeded, failed) + exec_profile : ExecProfile + python exec profile + inputs : dict + input parameters of the job + + Methods + ------- + prepare() -> None: + prepares the job + exec() -> None: + executes the job + eval() -> None: + evaluates the job + finalize() -> None: + finalizes the job + cancel() -> None: + cancels the job + """ + + def __init__( + self, + job_id: UUID, + job_status: JobStatusType, + exec_profile: ExecProfile, + inputs: dict, + ) -> None: + """ + Constructs all the necessary attributes for the python job object. + + Parameters + ---------- + job_id : uuid + id of the job + job_status : JobStatusType + current status of the job (eg. notstarted, succeeded, failed) + exec_profile : ExecProfile + python exec profile + inputs : dict + input parameters of the job + py_exec_session : PyExecSession + session in which the job will be run + """ + Job.__init__(self, job_id, job_status, exec_profile) + self.inputs = inputs + self.wf_lang = exec_profile.wf_lang + + def prepare(self) -> None: + """ + Prepares the job. + + Parameters + ---------- + + Returns + ------- + NONE + """ + + def exec(self) -> None: + """ + Executes the job. + + Parameters + ---------- + + Returns + ------- + NONE + """ + + def eval(self) -> None: + """ + Evaluates the job. + + Parameters + ---------- + + Returns + ------- + NONE + """ + + def finalize(self) -> None: + """ + Finalizes the job. + + Parameters + ---------- + + Returns + ------- + NONE + """ + + def cancel(self) -> None: + """ + Cancels the job. + + Parameters + ---------- + + Returns + ------- + NONE + """ diff --git a/exec_manager/wf_lang_type.py b/exec_manager/wf_lang_type.py new file mode 100644 index 0000000..7fd85f6 --- /dev/null +++ b/exec_manager/wf_lang_type.py @@ -0,0 +1,34 @@ +# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL +# for the German Human Genome-Phenome Archive (GHGA) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""enum for workflow languagae type""" + +from enum import Enum + + +class WfLangType(Enum): + """enumerate workflow language types""" + + CWL = "cwl" + """cwl language""" + + WDL = "wdl" + """wdl language""" + + NEXTFLOW = "nextflow" + """nextflow language""" + + SNAKEMAKE = "snakemake" + """snakemake language""" diff --git a/tests/unit/test_job_dao.py b/tests/unit/test_job_dao.py new file mode 100644 index 0000000..0e04a5d --- /dev/null +++ b/tests/unit/test_job_dao.py @@ -0,0 +1,130 @@ +# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL +# for the German Human Genome-Phenome Archive (GHGA) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +from exec_manager.dao.job_dao import create_job_dao, get_job, update_job_status +from exec_manager.exec_profile import ExecProfile +from exec_manager.exec_profile_type import ExecProfileType +from exec_manager.job_status_type import JobStatusType +from exec_manager.wf_lang_type import WfLangType + +# import pytest +# from exec_manager.dao.db_models import metadata +# from sqlalchemy import create_engine + + +# @pytest.fixture +# def example_engine(): +# engine = create_engine("sqlite+pysqlite://") +# metadata.create_all(engine) +# return engine + + +# @pytest.fixture +# def example_job_id(): +# return create_job_dao( +# JobStatusType.NOTSTARTET, +# ExecProfile(ExecProfileType.PYTHON, WfLangType.CWL), +# json.dumps({"test": 1}), +# {"hello": "world"}, +# ) + + +# engine = create_engine("sqlite+pysqlite://") +# metadata.create_all(engine) +job_status = JobStatusType.NOTSTARTET +exec_profile = ExecProfile(ExecProfileType.PYTHON, WfLangType.CWL) +workflow = {"test": 1} +inputs = {"hello": "world"} +example_job_id = create_job_dao( + JobStatusType.NOTSTARTET, + ExecProfile(ExecProfileType.PYTHON, WfLangType.CWL), + workflow, + inputs, +) + + +# @pytest.mark.usefixtures("example_engine") +def test_create_job_dao(): + job_id = create_job_dao(job_status, exec_profile, workflow, inputs) + db_job = get_job(job_id) + assert ( + str(db_job.job_id) == str(job_id) + and db_job.job_status.value == job_status.value + and ( + json.dumps( + { + "exec_profile_type": db_job.exec_profile.exec_profile_type.value, + "wf_lang": db_job.exec_profile.wf_lang.value, + } + ) + == json.dumps( + { + "exec_profile_type": ExecProfileType.PYTHON.value, + "wf_lang": WfLangType.CWL.value, + } + ) + ) + ) + + +# @pytest.mark.usefixtures("example_engine") +def test_get_job(): + job_id = example_job_id + db_job = get_job(job_id) + assert ( + str(db_job.job_id) == str(job_id) + and db_job.job_status.value == job_status.value + and ( + json.dumps( + { + "exec_profile_type": db_job.exec_profile.exec_profile_type.value, + "wf_lang": db_job.exec_profile.wf_lang.value, + } + ) + == json.dumps( + { + "exec_profile_type": ExecProfileType.PYTHON.value, + "wf_lang": WfLangType.CWL.value, + } + ) + ) + ) + + +# @pytest.mark.usefixtures("example_engine") +def test_update_job_status(): + job_id = example_job_id + update_job_status(job_id, JobStatusType.PREPARING) + db_job = get_job(job_id) + assert ( + str(db_job.job_id) == str(job_id) + and db_job.job_status.value == JobStatusType.PREPARING.value + and ( + json.dumps( + { + "exec_profile_type": db_job.exec_profile.exec_profile_type.value, + "wf_lang": db_job.exec_profile.wf_lang.value, + } + ) + == json.dumps( + { + "exec_profile_type": ExecProfileType.PYTHON.value, + "wf_lang": WfLangType.CWL.value, + } + ) + ) + ) From c286f37719b15bd509082acd29772dd558ac1ca2 Mon Sep 17 00:00:00 2001 From: KerstenBreuer Date: Thu, 19 May 2022 14:15:18 +0000 Subject: [PATCH 29/33] refactor --- .devcontainer/license_header.txt | 3 +- Dockerfile | 3 +- LICENSE | 3 +- exec_manager/__init__.py | 10 +- exec_manager/__main__.py | 3 +- exec_manager/dao/__init__.py | 5 +- exec_manager/dao/db_models.py | 7 +- exec_manager/dao/job_dao.py | 126 +++++++++++++++++-------- exec_manager/exec_profile.py | 22 ++--- exec_manager/exec_profile_type.py | 3 +- exec_manager/job.py | 19 ++-- exec_manager/job_factory.py | 12 +-- exec_manager/job_status_type.py | 9 +- exec_manager/py_exec_session.py | 3 +- exec_manager/python_job.py | 3 +- exec_manager/wf_lang_type.py | 3 +- scripts/get_package_name.py | 3 +- scripts/license_checker.py | 6 +- setup.cfg | 3 +- setup.py | 3 +- tests/__init__.py | 3 +- tests/fixtures/__init__.py | 3 +- tests/fixtures/utils.py | 3 +- tests/integration/__init__.py | 3 +- tests/integration/fixtures/__init__.py | 3 +- tests/integration/fixtures/utils.py | 3 +- tests/unit/__init__.py | 3 +- tests/unit/fixtures/__init__.py | 3 +- tests/unit/fixtures/utils.py | 3 +- tests/unit/test_dummy.py | 3 +- tests/unit/test_job_dao.py | 3 +- 31 files changed, 154 insertions(+), 128 deletions(-) diff --git a/.devcontainer/license_header.txt b/.devcontainer/license_header.txt index e193141..39f431d 100644 --- a/.devcontainer/license_header.txt +++ b/.devcontainer/license_header.txt @@ -1,5 +1,4 @@ -Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL -for the German Human Genome-Phenome Archive (GHGA) +Copyright 2021 - 2022 German Cancer Research Center (DKFZ) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/Dockerfile b/Dockerfile index 19f5604..1d7687d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,4 @@ -# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL -# for the German Human Genome-Phenome Archive (GHGA) +# Copyright 2021 - 2022 German Cancer Research Center (DKFZ) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/LICENSE b/LICENSE index edf280c..34ed0f0 100644 --- a/LICENSE +++ b/LICENSE @@ -186,8 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL - for the German Human Genome-Phenome Archive (GHGA) + Copyright 2021 - 2022 German Cancer Research Center (DKFZ) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/exec_manager/__init__.py b/exec_manager/__init__.py index 0921524..8faf810 100644 --- a/exec_manager/__init__.py +++ b/exec_manager/__init__.py @@ -1,5 +1,4 @@ -# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL -# for the German Human Genome-Phenome Archive (GHGA) +# Copyright 2021 - 2022 German Cancer Research Center (DKFZ) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""backend""" # Please adapt to package - +""" +A package managing execution of jobs in a way that is agnostic to +- the workflow execution environment +- the language used to describe the workflow +""" __version__ = "0.1.0" diff --git a/exec_manager/__main__.py b/exec_manager/__main__.py index 01ff8ce..4643630 100644 --- a/exec_manager/__main__.py +++ b/exec_manager/__main__.py @@ -1,5 +1,4 @@ -# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL -# for the German Human Genome-Phenome Archive (GHGA) +# Copyright 2021 - 2022 German Cancer Research Center (DKFZ) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/exec_manager/dao/__init__.py b/exec_manager/dao/__init__.py index 19e3303..82e3792 100644 --- a/exec_manager/dao/__init__.py +++ b/exec_manager/dao/__init__.py @@ -1,5 +1,4 @@ -# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL -# for the German Human Genome-Phenome Archive (GHGA) +# Copyright 2021 - 2022 German Cancer Research Center (DKFZ) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,3 +11,5 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +"""Package containing DAO classes.""" diff --git a/exec_manager/dao/db_models.py b/exec_manager/dao/db_models.py index 6c3ab62..4a05d6d 100644 --- a/exec_manager/dao/db_models.py +++ b/exec_manager/dao/db_models.py @@ -1,5 +1,4 @@ -# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL -# for the German Human Genome-Phenome Archive (GHGA) +# Copyright 2021 - 2022 German Cancer Research Center (DKFZ) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,7 +14,7 @@ """Defines all database specific ORM models""" - +import uuid from sqlalchemy import JSON, Column, String from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm.decl_api import DeclarativeMeta @@ -29,7 +28,7 @@ class DBJob(Base): __tablename__ = "job" - job_id = Column(String, primary_key=True) + job_id = Column(String, default=uuid.uuid4(), primary_key=True) job_status = Column(String, nullable=False) exec_profile = Column(JSON, nullable=False) workflow = Column(JSON, nullable=False) diff --git a/exec_manager/dao/job_dao.py b/exec_manager/dao/job_dao.py index f0f0c7f..ec39250 100644 --- a/exec_manager/dao/job_dao.py +++ b/exec_manager/dao/job_dao.py @@ -1,5 +1,4 @@ -# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL -# for the German Human Genome-Phenome Archive (GHGA) +# Copyright 2021 - 2022 German Cancer Research Center (DKFZ) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -28,9 +27,94 @@ from exec_manager.job_status_type import JobStatusType from exec_manager.wf_lang_type import WfLangType -DB_ENGINE = create_engine("sqlite+pysqlite://") -metadata.create_all(DB_ENGINE) +from abc import ABC, abstractmethod + +class JobDAO(ABC): + + @abstractmethod + def create( + self, + job_status: JobStatusType, + exec_profile: ExecProfile, + workflow: dict, + inputs: dict, + ) -> UUID: + """ + Inserts a job into the database. + + Parameters + ---------- + job_status: JobStatusType + current status of the job; initially it is JobStatusType.NOTSTARTED + exec_profile: ExecProfile + exec profile of this job + workflow + the jobs workflow + inputs: dict + the input parameters of the job + + Returns + ------- + UUID + """ + ... + + @abstractmethod + def get(self, job_id: str) -> Job: + ... + + @abstractmethod + def update(self, job_id: str, job: Job) -> None: + ... + +class SqlJobDAO(ABC): + + def __init__(self, db_url: str): + """Initialize DB.""" + self._engine = create_engine(db_url) + metadata.create_all(self._engine) + + @abstractmethod + def create( + self, + job_status: JobStatusType, + exec_profile: ExecProfile, + workflow: dict, + inputs: dict, + ) -> UUID: + """ + Inserts a job into the database. + + Parameters + ---------- + job_status: JobStatusType + current status of the job; initially it is JobStatusType.NOTSTARTED + exec_profile: ExecProfile + exec profile of this job + workflow + the jobs workflow + inputs: dict + the input parameters of the job + + Returns + ------- + UUID + """ + with self._engine.connect() as connection: + connection.execute( + insert(DBJob.__table__).values( + job_status=job_status.value, + exec_profile=exec_profile.dict(), + workflow=workflow, + inputs=inputs, + ) + ) + return job_id + + @abstractmethod + def get(job_id) -> Job: + # another sql query here def create_job_dao( job_status: JobStatusType, @@ -59,23 +143,6 @@ def create_job_dao( ------- UUID """ - job_id = generate_job_id() - job_id_str = str(job_id) - job_status_str = job_status.value - exec_profile_json = json.dumps( - { - "exec_profile_type": exec_profile.exec_profile_type.value, - "wf_lang": exec_profile.wf_lang.value, - } - ) - inputs_json = json.dumps(inputs) - with db_engine.connect() as connection: - connection.execute( - insert(DBJob.__table__).values( - (job_id_str, job_status_str, exec_profile_json, workflow, inputs_json) - ) - ) - return job_id def get_job(job_id: UUID, db_engine: Engine = DB_ENGINE) -> Job: @@ -134,20 +201,3 @@ def update_job_status( .where(DBJob.job_id == str(job_id)) .values(job_status=new_job_status.value) ) - - -def generate_job_id() -> UUID: - """ - Generates a unique job id. - - Parameters - ---------- - - Returns - ------- - UUID - """ - job_id = uuid4() - # while get_job(job_id) is not None: - # job_id = uuid4() - return job_id diff --git a/exec_manager/exec_profile.py b/exec_manager/exec_profile.py index b7a8f63..2f229bc 100644 --- a/exec_manager/exec_profile.py +++ b/exec_manager/exec_profile.py @@ -1,5 +1,4 @@ -# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL -# for the German Human Genome-Phenome Archive (GHGA) +# Copyright 2021 - 2022 German Cancer Research Center (DKFZ) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,11 +14,13 @@ """class for exec profile""" +from pydantic import BaseModel + from exec_manager.exec_profile_type import ExecProfileType from exec_manager.wf_lang_type import WfLangType -class ExecProfile: +class ExecProfile(BaseModel): """ class for Exec-Profile @@ -36,16 +37,5 @@ class for Exec-Profile ------- """ - def __init__(self, exec_profile_type: ExecProfileType, wf_lang: WfLangType) -> None: - """ - Constructs all the necessary attributes for the exec profile object. - - Parameters - ---------- - exec_profile_type : ExecProfileType - type of the exec profile (bash, python, wes) - wf_lang : WfLangType - workflow language (cwl, wdl, nextflow, snakemake) - """ - self.exec_profile_type = exec_profile_type - self.wf_lang = wf_lang + type_: ExecProfileType + wf_lang: WfLangType diff --git a/exec_manager/exec_profile_type.py b/exec_manager/exec_profile_type.py index 5e7637a..ed9ce96 100644 --- a/exec_manager/exec_profile_type.py +++ b/exec_manager/exec_profile_type.py @@ -1,5 +1,4 @@ -# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL -# for the German Human Genome-Phenome Archive (GHGA) +# Copyright 2021 - 2022 German Cancer Research Center (DKFZ) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/exec_manager/job.py b/exec_manager/job.py index ab5fb8c..3b6d01e 100644 --- a/exec_manager/job.py +++ b/exec_manager/job.py @@ -1,5 +1,4 @@ -# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL -# for the German Human Genome-Phenome Archive (GHGA) +# Copyright 2021 - 2022 German Cancer Research Center (DKFZ) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,13 +14,14 @@ """class for job""" +from abc import ABC, abstractmethod from uuid import UUID from exec_manager.exec_profile import ExecProfile from exec_manager.job_status_type import JobStatusType -class Job: +class Job(ABC): """ class for job @@ -69,9 +69,7 @@ def __init__( self.job_status = job_status self.exec_profile = exec_profile - # def __hash__(self) -> int: - # return hash((self.job_id, self.job_status, self.exec_profile)) - + @abstractmethod def prepare(self) -> None: """ Prepares the job. @@ -83,7 +81,9 @@ def prepare(self) -> None: ------- NONE """ + ... + @abstractmethod def exec(self) -> None: """ Executes the job. @@ -95,7 +95,9 @@ def exec(self) -> None: ------- NONE """ + ... + @abstractmethod def eval(self) -> None: """ Evaluates the job. @@ -107,7 +109,9 @@ def eval(self) -> None: ------- NONE """ + ... + @abstractmethod def finalize(self) -> None: """ Finalizes the job. @@ -119,7 +123,9 @@ def finalize(self) -> None: ------- NONE """ + ... + @abstractmethod def cancel(self) -> None: """ Cancels the job. @@ -131,3 +137,4 @@ def cancel(self) -> None: ------- NONE """ + ... diff --git a/exec_manager/job_factory.py b/exec_manager/job_factory.py index b9ffd35..71c78de 100644 --- a/exec_manager/job_factory.py +++ b/exec_manager/job_factory.py @@ -1,5 +1,4 @@ -# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL -# for the German Human Genome-Phenome Archive (GHGA) +# Copyright 2021 - 2022 German Cancer Research Center (DKFZ) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -45,8 +44,7 @@ def create_job( job_id = create_dao_job(job_status, exec_profile, workflow, inputs) if exec_profile.exec_profile_type == ExecProfileType.PYTHON: return PythonJob(job_id, job_status, exec_profile, inputs) - if exec_profile.exec_profile_type == ExecProfileType.BASH: - pass # place for bash job - if exec_profile.exec_profile_type == ExecProfileType.WES: - pass # place for wes job - return Job(job_id, job_status, exec_profile) + elif exec_profile.exec_profile_type == ExecProfileType.BASH: + raise NotImplementedError("Execution profiles of type Bash not supported, yet") + else: + raise NotImplementedError("Execution profiles of type WES not supported, yet") diff --git a/exec_manager/job_status_type.py b/exec_manager/job_status_type.py index afbd87e..bfdfc9c 100644 --- a/exec_manager/job_status_type.py +++ b/exec_manager/job_status_type.py @@ -1,5 +1,4 @@ -# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL -# for the German Human Genome-Phenome Archive (GHGA) +# Copyright 2021 - 2022 German Cancer Research Center (DKFZ) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,7 +18,11 @@ class JobStatusType(Enum): - """enumerate job status types""" + """Enumerate job status types: + - NOTSTARTET: job ist not started yet + - ... + + """ NOTSTARTET = "not started" """job ist not started yet""" diff --git a/exec_manager/py_exec_session.py b/exec_manager/py_exec_session.py index bb6e616..c32ef79 100644 --- a/exec_manager/py_exec_session.py +++ b/exec_manager/py_exec_session.py @@ -1,5 +1,4 @@ -# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL -# for the German Human Genome-Phenome Archive (GHGA) +# Copyright 2021 - 2022 German Cancer Research Center (DKFZ) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/exec_manager/python_job.py b/exec_manager/python_job.py index 421eeb4..2ed135e 100644 --- a/exec_manager/python_job.py +++ b/exec_manager/python_job.py @@ -1,5 +1,4 @@ -# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL -# for the German Human Genome-Phenome Archive (GHGA) +# Copyright 2021 - 2022 German Cancer Research Center (DKFZ) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/exec_manager/wf_lang_type.py b/exec_manager/wf_lang_type.py index 7fd85f6..6fa7774 100644 --- a/exec_manager/wf_lang_type.py +++ b/exec_manager/wf_lang_type.py @@ -1,5 +1,4 @@ -# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL -# for the German Human Genome-Phenome Archive (GHGA) +# Copyright 2021 - 2022 German Cancer Research Center (DKFZ) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/scripts/get_package_name.py b/scripts/get_package_name.py index f43e467..23ec881 100755 --- a/scripts/get_package_name.py +++ b/scripts/get_package_name.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL -# for the German Human Genome-Phenome Archive (GHGA) +# Copyright 2021 - 2022 German Cancer Research Center (DKFZ) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/scripts/license_checker.py b/scripts/license_checker.py index b85be3f..2608590 100755 --- a/scripts/license_checker.py +++ b/scripts/license_checker.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL -# for the German Human Genome-Phenome Archive (GHGA) +# Copyright 2021 - 2022 German Cancer Research Center (DKFZ) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -93,8 +92,7 @@ # A list of all chars that may be used to introduce a comment: COMMENT_CHARS = ["#"] -AUTHOR = """Universität Tübingen, DKFZ and EMBL -for the German Human Genome-Phenome Archive (GHGA)""" +AUTHOR = """German Cancer Research Center (DKFZ)""" # The copyright notice should not date earlier than this year: MIN_YEAR = 2021 diff --git a/setup.cfg b/setup.cfg index caab8e3..f0786f3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,4 @@ -# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL -# for the German Human Genome-Phenome Archive (GHGA) +# Copyright 2021 - 2022 German Cancer Research Center (DKFZ) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/setup.py b/setup.py index 1766b45..f56b210 100755 --- a/setup.py +++ b/setup.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL -# for the German Human Genome-Phenome Archive (GHGA) +# Copyright 2021 - 2022 German Cancer Research Center (DKFZ) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/__init__.py b/tests/__init__.py index 93766df..62f385b 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,5 +1,4 @@ -# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL -# for the German Human Genome-Phenome Archive (GHGA) +# Copyright 2021 - 2022 German Cancer Research Center (DKFZ) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/fixtures/__init__.py b/tests/fixtures/__init__.py index dc01178..81a2573 100644 --- a/tests/fixtures/__init__.py +++ b/tests/fixtures/__init__.py @@ -1,5 +1,4 @@ -# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL -# for the German Human Genome-Phenome Archive (GHGA) +# Copyright 2021 - 2022 German Cancer Research Center (DKFZ) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/fixtures/utils.py b/tests/fixtures/utils.py index a38f1a9..154491a 100644 --- a/tests/fixtures/utils.py +++ b/tests/fixtures/utils.py @@ -1,5 +1,4 @@ -# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL -# for the German Human Genome-Phenome Archive (GHGA) +# Copyright 2021 - 2022 German Cancer Research Center (DKFZ) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 00db53a..500ae8e 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -1,5 +1,4 @@ -# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL -# for the German Human Genome-Phenome Archive (GHGA) +# Copyright 2021 - 2022 German Cancer Research Center (DKFZ) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/integration/fixtures/__init__.py b/tests/integration/fixtures/__init__.py index 8031adf..91891cb 100644 --- a/tests/integration/fixtures/__init__.py +++ b/tests/integration/fixtures/__init__.py @@ -1,5 +1,4 @@ -# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL -# for the German Human Genome-Phenome Archive (GHGA) +# Copyright 2021 - 2022 German Cancer Research Center (DKFZ) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/integration/fixtures/utils.py b/tests/integration/fixtures/utils.py index a38f1a9..154491a 100644 --- a/tests/integration/fixtures/utils.py +++ b/tests/integration/fixtures/utils.py @@ -1,5 +1,4 @@ -# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL -# for the German Human Genome-Phenome Archive (GHGA) +# Copyright 2021 - 2022 German Cancer Research Center (DKFZ) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py index e4011ec..1c514b4 100644 --- a/tests/unit/__init__.py +++ b/tests/unit/__init__.py @@ -1,5 +1,4 @@ -# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL -# for the German Human Genome-Phenome Archive (GHGA) +# Copyright 2021 - 2022 German Cancer Research Center (DKFZ) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/unit/fixtures/__init__.py b/tests/unit/fixtures/__init__.py index dea2cf7..4d86eb7 100644 --- a/tests/unit/fixtures/__init__.py +++ b/tests/unit/fixtures/__init__.py @@ -1,5 +1,4 @@ -# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL -# for the German Human Genome-Phenome Archive (GHGA) +# Copyright 2021 - 2022 German Cancer Research Center (DKFZ) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/unit/fixtures/utils.py b/tests/unit/fixtures/utils.py index a38f1a9..154491a 100644 --- a/tests/unit/fixtures/utils.py +++ b/tests/unit/fixtures/utils.py @@ -1,5 +1,4 @@ -# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL -# for the German Human Genome-Phenome Archive (GHGA) +# Copyright 2021 - 2022 German Cancer Research Center (DKFZ) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/unit/test_dummy.py b/tests/unit/test_dummy.py index dbd2856..ee6d874 100644 --- a/tests/unit/test_dummy.py +++ b/tests/unit/test_dummy.py @@ -1,5 +1,4 @@ -# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL -# for the German Human Genome-Phenome Archive (GHGA) +# Copyright 2021 - 2022 German Cancer Research Center (DKFZ) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/unit/test_job_dao.py b/tests/unit/test_job_dao.py index 0e04a5d..8288d16 100644 --- a/tests/unit/test_job_dao.py +++ b/tests/unit/test_job_dao.py @@ -1,5 +1,4 @@ -# Copyright 2021 - 2022 Universität Tübingen, DKFZ and EMBL -# for the German Human Genome-Phenome Archive (GHGA) +# Copyright 2021 - 2022 German Cancer Research Center (DKFZ) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 937093009f83ff901e1f3fc3f7f9c96715d1eaf1 Mon Sep 17 00:00:00 2001 From: e-buerger Date: Mon, 23 May 2022 09:13:05 +0200 Subject: [PATCH 30/33] restructure modules --- exec_manager/dao/db_models.py | 1 + exec_manager/dao/job_dao.py | 215 +++++----- exec_manager/exec_profile_type.py | 30 -- .../{exec_profile.py => exec_profiles.py} | 19 +- exec_manager/job.py | 140 ------- exec_manager/job_factory.py | 50 --- exec_manager/job_status_type.py | 49 --- exec_manager/jobs.py | 387 ++++++++++++++++++ exec_manager/py_exec_session.py | 86 ---- exec_manager/python_job.py | 141 ------- exec_manager/{wf_lang_type.py => utils.py} | 20 +- setup.cfg | 1 + tests/unit/test_job_dao.py | 210 ++++++---- 13 files changed, 644 insertions(+), 705 deletions(-) delete mode 100644 exec_manager/exec_profile_type.py rename exec_manager/{exec_profile.py => exec_profiles.py} (75%) delete mode 100644 exec_manager/job.py delete mode 100644 exec_manager/job_factory.py delete mode 100644 exec_manager/job_status_type.py create mode 100644 exec_manager/jobs.py delete mode 100644 exec_manager/py_exec_session.py delete mode 100644 exec_manager/python_job.py rename exec_manager/{wf_lang_type.py => utils.py} (76%) diff --git a/exec_manager/dao/db_models.py b/exec_manager/dao/db_models.py index 4a05d6d..be125fa 100644 --- a/exec_manager/dao/db_models.py +++ b/exec_manager/dao/db_models.py @@ -15,6 +15,7 @@ """Defines all database specific ORM models""" import uuid + from sqlalchemy import JSON, Column, String from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm.decl_api import DeclarativeMeta diff --git a/exec_manager/dao/job_dao.py b/exec_manager/dao/job_dao.py index ec39250..f7ee3b3 100644 --- a/exec_manager/dao/job_dao.py +++ b/exec_manager/dao/job_dao.py @@ -12,25 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""class for job dao""" +"""module for job dao""" import json -from uuid import UUID, uuid4 +from abc import ABC, abstractmethod +from uuid import UUID from sqlalchemy import create_engine, insert, select, update -from sqlalchemy.engine import Engine from exec_manager.dao.db_models import DBJob, metadata -from exec_manager.exec_profile import ExecProfile -from exec_manager.exec_profile_type import ExecProfileType -from exec_manager.job import Job -from exec_manager.job_status_type import JobStatusType -from exec_manager.wf_lang_type import WfLangType +from exec_manager.exec_profiles import ExecProfile, ExecProfileType +from exec_manager.jobs import Job, JobStatusType, PythonJob +from exec_manager.utils import WfLangType -from abc import ABC, abstractmethod class JobDAO(ABC): - + """abstract class for job dao""" + @abstractmethod def create( self, @@ -60,21 +58,48 @@ def create( ... @abstractmethod - def get(self, job_id: str) -> Job: - ... + def get(self, job_id: UUID) -> Job: + """ + Returns a job by his job id. + + Parameters + ---------- + job_id: UUID + id of the job + + Returns + ------- + Job + """ + ... @abstractmethod - def update(self, job_id: str, job: Job) -> None: + def update(self, job_id: UUID, job: Job) -> None: + """ + Updates a jobs by his id. + + Parameters + ---------- + job_id: UUID + id of the job + job: Job + updated job + + Returns + ------- + None + """ ... -class SqlJobDAO(ABC): - + +class SQLJobDAO(JobDAO): + """class for sql job dao""" + def __init__(self, db_url: str): """Initialize DB.""" self._engine = create_engine(db_url) metadata.create_all(self._engine) - @abstractmethod def create( self, job_status: JobStatusType, @@ -91,7 +116,7 @@ def create( current status of the job; initially it is JobStatusType.NOTSTARTED exec_profile: ExecProfile exec profile of this job - workflow + workflow: dict the jobs workflow inputs: dict the input parameters of the job @@ -101,103 +126,77 @@ def create( UUID """ with self._engine.connect() as connection: - connection.execute( - insert(DBJob.__table__).values( + cursor = connection.execute( + insert(DBJob.__table__) + .values( job_status=job_status.value, exec_profile=exec_profile.dict(), workflow=workflow, inputs=inputs, ) + .returning(DBJob.job_id) ) - return job_id + result = cursor.fetchall + return result[0][0] # job_id + def get(self, job_id: UUID) -> Job: + """ + Returns a job by his job id. - @abstractmethod - def get(job_id) -> Job: - # another sql query here - -def create_job_dao( - job_status: JobStatusType, - exec_profile: ExecProfile, - workflow: dict, - inputs: dict, - db_engine: Engine = DB_ENGINE, -) -> UUID: - """ - Inserts a job into the database. - - Parameters - ---------- - job_status: JobStatusType - current status of the job; initially it is JobStatusType.NOTSTARTED - exec_profile: ExecProfile - exec profile of this job - workflow - the jobs workflow - inputs: dict - the input parameters of the job - engine: engine - db engine where the connection will be established (default is sqlite with pysqlite) - - Returns - ------- - UUID - """ - - -def get_job(job_id: UUID, db_engine: Engine = DB_ENGINE) -> Job: - """ - Returns a job by his job id. - - Parameters - ---------- - job_id: UUID - id of the job - engine: engine - db engine where the connection will be established (default is sqlite with pysqlite) - - Returns - ------- - Job - """ - with db_engine.connect() as connection: - cursor = connection.execute( - select([DBJob.job_id, DBJob.job_status, DBJob.exec_profile]).where( - DBJob.job_id == str(job_id) + Parameters + ---------- + job_id: UUID + id of the job + + Returns + ------- + Job + """ + with self._engine.connect() as connection: + cursor = connection.execute( + select([DBJob.job_id, DBJob.job_status, DBJob.exec_profile]).where( + DBJob.job_id == str(job_id) + ) + ) + result = cursor.fetchall() + job_status = JobStatusType(result[0][1]) + exec_profile = json.loads(result[0][2]) + exec_profile = ExecProfile( + ExecProfileType(exec_profile["exec_profile_type"]), + WfLangType(exec_profile["wf_lang"]), + ) + inputs = json.loads(result[0][4]) + if exec_profile.type_ == ExecProfileType.PYTHON: + return PythonJob(job_id, job_status, exec_profile, inputs) + if exec_profile.exec_profile_type == ExecProfileType.BASH: + raise NotImplementedError( + "Execution profiles of type Bash not supported, yet" + ) + raise NotImplementedError( + "Execution profiles of type WES not supported, yet" + ) + + def update(self, job_id: UUID, job: Job) -> None: + """ + Updates a jobs by his id. + + Parameters + ---------- + job_id: UUID + id of the job + job: Job + updated job + + Returns + ------- + None + """ + with self._engine.connect() as connection: + connection.execute( + update(DBJob.__table__) + .where(DBJob.job_id == str(job_id)) + .values( + job_status=job.job_status.value, + exec_profile=job.exec_profile.dict(), + ) ) - ) - result = cursor.fetchall() - job_status = JobStatusType(result[0][1]) - exec_profile = json.loads(result[0][2]) - exec_profile = ExecProfile( - ExecProfileType(exec_profile["exec_profile_type"]), - WfLangType(exec_profile["wf_lang"]), - ) - return Job(job_id, job_status, exec_profile) - - -def update_job_status( - job_id: UUID, new_job_status: JobStatusType, db_engine: Engine = DB_ENGINE -) -> None: - """ - Updates a jobs status by his job id. - - Parameters - ---------- - job_id: UUID - id of the job - new_job_status: JobStatusType - new status of the job; cannot be JobStatusType.NOTSTARTED - engine: engine - db engine where the connection will be established (default is sqlite with pysqlite) - - Returns - ------- - None - """ - with db_engine.connect() as connection: - connection.execute( - update(DBJob.__table__) - .where(DBJob.job_id == str(job_id)) - .values(job_status=new_job_status.value) - ) diff --git a/exec_manager/exec_profile_type.py b/exec_manager/exec_profile_type.py deleted file mode 100644 index ed9ce96..0000000 --- a/exec_manager/exec_profile_type.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2021 - 2022 German Cancer Research Center (DKFZ) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""enum for exec profile type""" - -from enum import Enum - - -class ExecProfileType(Enum): - """enumaerate exec profile types""" - - BASH = "bash" - """execution with bash""" - - PYTHON = "python" - """execution with python""" - - WES = "wes" - """execution with wes""" diff --git a/exec_manager/exec_profile.py b/exec_manager/exec_profiles.py similarity index 75% rename from exec_manager/exec_profile.py rename to exec_manager/exec_profiles.py index 2f229bc..20aa9f5 100644 --- a/exec_manager/exec_profile.py +++ b/exec_manager/exec_profiles.py @@ -12,12 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""class for exec profile""" +"""module for execution profiles""" + +from enum import Enum from pydantic import BaseModel -from exec_manager.exec_profile_type import ExecProfileType -from exec_manager.wf_lang_type import WfLangType +from exec_manager.utils import WfLangType + + +class ExecProfileType(Enum): + """Enumerate exec profile types: + - BASH: execution with bash + - PYTHON: execution with python + - WES: execution with wes + """ + + BASH = "bash" + PYTHON = "python" + WES = "wes" class ExecProfile(BaseModel): diff --git a/exec_manager/job.py b/exec_manager/job.py deleted file mode 100644 index 3b6d01e..0000000 --- a/exec_manager/job.py +++ /dev/null @@ -1,140 +0,0 @@ -# Copyright 2021 - 2022 German Cancer Research Center (DKFZ) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""class for job""" - -from abc import ABC, abstractmethod -from uuid import UUID - -from exec_manager.exec_profile import ExecProfile -from exec_manager.job_status_type import JobStatusType - - -class Job(ABC): - """ - class for job - - ... - - Attributes - ---------- - job_id : uuid - id of the job - job_status : JobStatusType - current status of the job (eg. notstarted, succeeded, failed) - exec_profile : ExecProfile - exec profile with which the job should be executed - - Methods - ------- - prepare() -> None: - prepares the job - exec() -> None: - executes the job - eval() -> None: - evaluates the job - finalize() -> None: - finalizes the job - cancel() -> None: - cancels the job - """ - - def __init__( - self, job_id: UUID, job_status: JobStatusType, exec_profile: ExecProfile - ) -> None: - """ - Constructs all the necessary attributes for the job object. - - Parameters - ---------- - job_id : uuid - id of the job - job_status : JobStatusType - current status of the job (eg. notstarted, succeeded, failed) - exec_profile : ExecProfile - exec profile with which the job should be executed - """ - self.job_id = job_id - self.job_status = job_status - self.exec_profile = exec_profile - - @abstractmethod - def prepare(self) -> None: - """ - Prepares the job. - - Parameters - ---------- - - Returns - ------- - NONE - """ - ... - - @abstractmethod - def exec(self) -> None: - """ - Executes the job. - - Parameters - ---------- - - Returns - ------- - NONE - """ - ... - - @abstractmethod - def eval(self) -> None: - """ - Evaluates the job. - - Parameters - ---------- - - Returns - ------- - NONE - """ - ... - - @abstractmethod - def finalize(self) -> None: - """ - Finalizes the job. - - Parameters - ---------- - - Returns - ------- - NONE - """ - ... - - @abstractmethod - def cancel(self) -> None: - """ - Cancels the job. - - Parameters - ---------- - - Returns - ------- - NONE - """ - ... diff --git a/exec_manager/job_factory.py b/exec_manager/job_factory.py deleted file mode 100644 index 71c78de..0000000 --- a/exec_manager/job_factory.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2021 - 2022 German Cancer Research Center (DKFZ) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""class for job factory""" - -from typing import Callable - -from exec_manager.dao.job_dao import create_job_dao -from exec_manager.exec_profile import ExecProfile -from exec_manager.exec_profile_type import ExecProfileType -from exec_manager.job import Job -from exec_manager.job_status_type import JobStatusType -from exec_manager.python_job import PythonJob - - -def create_job( - inputs: dict, - workflow, - exec_profile: ExecProfile, - create_dao_job: Callable = create_job_dao, -) -> Job: - """ - Creates a job. - - Parameters - ---------- - - Returns - ------- - Job - """ - job_status = JobStatusType.NOTSTARTET - job_id = create_dao_job(job_status, exec_profile, workflow, inputs) - if exec_profile.exec_profile_type == ExecProfileType.PYTHON: - return PythonJob(job_id, job_status, exec_profile, inputs) - elif exec_profile.exec_profile_type == ExecProfileType.BASH: - raise NotImplementedError("Execution profiles of type Bash not supported, yet") - else: - raise NotImplementedError("Execution profiles of type WES not supported, yet") diff --git a/exec_manager/job_status_type.py b/exec_manager/job_status_type.py deleted file mode 100644 index bfdfc9c..0000000 --- a/exec_manager/job_status_type.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2021 - 2022 German Cancer Research Center (DKFZ) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""enum for job status type""" - -from enum import Enum - - -class JobStatusType(Enum): - """Enumerate job status types: - - NOTSTARTET: job ist not started yet - - ... - - """ - - NOTSTARTET = "not started" - """job ist not started yet""" - - PREPARING = "preparing" - """job is preparing""" - - EXECUTING = "executing" - """job is executing""" - - EVALUATING = "evaluating" - """job ist evaluating""" - - FINALZING = "finalizing" - """job is finalizing""" - - CANCELED = "canceled" - """job is canceled""" - - FAILED = "failed" - """job is failed""" - - SUCCEEDED = "succeeded" - """job is succeeded""" diff --git a/exec_manager/jobs.py b/exec_manager/jobs.py new file mode 100644 index 0000000..06b2426 --- /dev/null +++ b/exec_manager/jobs.py @@ -0,0 +1,387 @@ +# Copyright 2021 - 2022 German Cancer Research Center (DKFZ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""module for jobs""" + +from abc import ABC, abstractmethod +from enum import Enum +from typing import Callable +from uuid import UUID + +from exec_manager.dao.job_dao import SQLJobDAO +from exec_manager.exec_profiles import ExecProfile, ExecProfileType + + +class JobStatusType(Enum): + """Enumerate job status types: + - NOTSTARTET: job is not started yet + - PREPARING: job is preparing + - EXECUTING: job is executing + - EVALUATING: job is evaluating + - FINALZING: job is finalizing + - CANCELED: job is canceled + - FAILED: job is failed + - SUCCEEDED: job is succeeded + """ + + NOTSTARTET = "not started" + PREPARING = "preparing" + EXECUTING = "executing" + EVALUATING = "evaluating" + FINALZING = "finalizing" + CANCELED = "canceled" + FAILED = "failed" + SUCCEEDED = "succeeded" + + +class Job(ABC): + """ + class for job + + ... + + Attributes + ---------- + job_id : uuid + id of the job + job_status : JobStatusType + current status of the job (eg. notstarted, succeeded, failed) + exec_profile : ExecProfile + exec profile with which the job should be executed + + Methods + ------- + prepare() -> None: + prepares the job + exec() -> None: + executes the job + eval() -> None: + evaluates the job + finalize() -> None: + finalizes the job + cancel() -> None: + cancels the job + """ + + def __init__( + self, job_id: UUID, job_status: JobStatusType, exec_profile: ExecProfile + ) -> None: + """ + Constructs all the necessary attributes for the job object. + + Parameters + ---------- + job_id : UUID + id of the job + job_status : JobStatusType + current status of the job (eg. notstarted, succeeded, failed, ...) + exec_profile : ExecProfile + exec profile with which the job should be executed (bash, python, WES) + """ + self.job_id = job_id + self.job_status = job_status + self.exec_profile = exec_profile + + @abstractmethod + def prepare(self) -> None: + """ + Prepares the job. + + Parameters + ---------- + + Returns + ------- + NONE + """ + ... + + @abstractmethod + def exec(self) -> None: + """ + Executes the job. + + Parameters + ---------- + + Returns + ------- + NONE + """ + ... + + @abstractmethod + def eval(self) -> None: + """ + Evaluates the job. + + Parameters + ---------- + + Returns + ------- + NONE + """ + ... + + @abstractmethod + def finalize(self) -> None: + """ + Finalizes the job. + + Parameters + ---------- + + Returns + ------- + NONE + """ + ... + + @abstractmethod + def cancel(self) -> None: + """ + Cancels the job. + + Parameters + ---------- + + Returns + ------- + NONE + """ + ... + + +class PythonJob(Job): + """ + class for python job + + ... + + Attributes + ---------- + job_id : UUID + id of the job + job_status : JobStatusType + current status of the job (eg. notstarted, succeeded, failed) + exec_profile : ExecProfile + python exec profile + inputs : dict + input parameters of the job + + Methods + ------- + prepare() -> None: + prepares the job + exec() -> None: + executes the job + eval() -> None: + evaluates the job + finalize() -> None: + finalizes the job + cancel() -> None: + cancels the job + """ + + def __init__( + self, + job_id: UUID, + job_status: JobStatusType, + exec_profile: ExecProfile, + inputs: dict, + ) -> None: + """ + Constructs all the necessary attributes for the python job object. + + Parameters + ---------- + job_id : uuid + id of the job + job_status : JobStatusType + current status of the job (eg. notstarted, succeeded, failed) + exec_profile : ExecProfile + python exec profile + inputs : dict + input parameters of the job + """ + Job.__init__(self, job_id, job_status, exec_profile) + self.inputs = inputs + self.wf_lang = exec_profile.wf_lang + + def prepare(self) -> None: + """ + Prepares the job. + + Parameters + ---------- + + Returns + ------- + NONE + """ + + def exec(self) -> None: + """ + Executes the job. + + Parameters + ---------- + + Returns + ------- + NONE + """ + + def eval(self) -> None: + """ + Evaluates the job. + + Parameters + ---------- + + Returns + ------- + NONE + """ + + def finalize(self) -> None: + """ + Finalizes the job. + + Parameters + ---------- + + Returns + ------- + NONE + """ + + def cancel(self) -> None: + """ + Cancels the job. + + Parameters + ---------- + + Returns + ------- + NONE + """ + + +class PyExecSession: + """ + class for python job + + ... + + Attributes + ---------- + max_retries : int + + Methods + ------- + run() -> None + runs a job + """ + + def __init__( + self, + max_retries: int = 0, + ) -> None: + """ + Constructs all the necessary attributes for the python exec session. + + Parameters + ---------- + max_retries : int + number of maximum retries when the execution fails (default: 0) + """ + self._max_retries = max_retries + + def run( + self, + job_id: UUID, + job_status: JobStatusType, + exec_profile: ExecProfile, + inputs: dict, + ) -> None: + """ + runs a job. + + Parameters + ---------- + job_id: UUID + id of the job + job_status: JobStatusType + current status of the job (e.g. notstarted, executing, failed, ...) + exec_profile: ExecProfile + exec profile with which the job should be executed (bash, python, WES) + inputs: dict, + + Returns + ------- + NONE + """ + counter = -1 + sql_job_dao = SQLJobDAO("sqlite+pysqlite://") + while self._max_retries > counter: + python_job = PythonJob(job_id, job_status, exec_profile, inputs) + python_job.job_status = JobStatusType.PREPARING + sql_job_dao.update(job_id, python_job) + python_job.prepare() + python_job.job_status = JobStatusType.EXECUTING + sql_job_dao.update(job_id, python_job) + python_job.exec() + python_job.job_status = JobStatusType.EVALUATING + sql_job_dao.update(job_id, python_job) + python_job.eval() + python_job.job_status = JobStatusType.FINALZING + sql_job_dao.update(job_id, python_job) + python_job.finalize() + if sql_job_dao.get(job_id).job_status == JobStatusType.SUCCEEDED: + break + counter = counter + 1 + + +def create_job( + inputs: dict, + workflow: dict, + exec_profile: ExecProfile, + create: Callable = SQLJobDAO.create, +) -> Job: + """ + Creates a job. + + Parameters + ---------- + inputs: dict + input paramters of the job + workflow: dict + the job's workflow + exec_profile: ExecProfile + exec profile with which the job should be executed (bash, python, WES) + + Returns + ------- + Job + """ + job_status = JobStatusType.NOTSTARTET + job_id = create(job_status, exec_profile, workflow, inputs) + if exec_profile.exec_profile_type == ExecProfileType.PYTHON: + return PythonJob(job_id, job_status, exec_profile, inputs) + if exec_profile.exec_profile_type == ExecProfileType.BASH: + raise NotImplementedError("Execution profiles of type Bash not supported, yet") + raise NotImplementedError("Execution profiles of type WES not supported, yet") diff --git a/exec_manager/py_exec_session.py b/exec_manager/py_exec_session.py deleted file mode 100644 index c32ef79..0000000 --- a/exec_manager/py_exec_session.py +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright 2021 - 2022 German Cancer Research Center (DKFZ) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""class for python exec session""" - - -from uuid import UUID - -from exec_manager.dao.job_dao import get_job, update_job_status -from exec_manager.exec_profile import ExecProfile -from exec_manager.job_status_type import JobStatusType -from exec_manager.python_job import PythonJob - - -class PyExecSession: - """ - class for python job - - ... - - Attributes - ---------- - max_retries : int - - Methods - ------- - start() -> None - starts a session - """ - - def __init__( - self, - max_retries: int = 0, - ) -> None: - """ - Constructs all the necessary attributes for the python exec session. - - Parameters - ---------- - max_retries : int - number of maximum retries when the execution fails - """ - self.max_retries = max_retries - - def run( - self, - job_id: UUID, - job_status: JobStatusType, - exec_profile: ExecProfile, - inputs: dict, - ) -> None: - """ - Starts the session. - - Parameters - ---------- - - Returns - ------- - NONE - """ - counter = -1 - while self.max_retries > counter: - python_job = PythonJob(job_id, job_status, exec_profile, inputs) - update_job_status(job_id, JobStatusType.PREPARING) - python_job.prepare() - update_job_status(job_id, JobStatusType.EXECUTING) - python_job.exec() - update_job_status(job_id, JobStatusType.EVALUATING) - python_job.eval() - update_job_status(job_id, JobStatusType.FINALZING) - python_job.finalize() - if get_job(job_id).job_status == JobStatusType.SUCCEEDED: - break - counter = counter + 1 diff --git a/exec_manager/python_job.py b/exec_manager/python_job.py deleted file mode 100644 index 2ed135e..0000000 --- a/exec_manager/python_job.py +++ /dev/null @@ -1,141 +0,0 @@ -# Copyright 2021 - 2022 German Cancer Research Center (DKFZ) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""class for python job""" - -# from subprocess import Popen # nosec -from uuid import UUID - -from exec_manager.exec_profile import ExecProfile -from exec_manager.job import Job -from exec_manager.job_status_type import JobStatusType - - -class PythonJob(Job): - """ - class for python job - - ... - - Attributes - ---------- - job_id : UUID - id of the job - job_status : JobStatusType - current status of the job (eg. notstarted, succeeded, failed) - exec_profile : ExecProfile - python exec profile - inputs : dict - input parameters of the job - - Methods - ------- - prepare() -> None: - prepares the job - exec() -> None: - executes the job - eval() -> None: - evaluates the job - finalize() -> None: - finalizes the job - cancel() -> None: - cancels the job - """ - - def __init__( - self, - job_id: UUID, - job_status: JobStatusType, - exec_profile: ExecProfile, - inputs: dict, - ) -> None: - """ - Constructs all the necessary attributes for the python job object. - - Parameters - ---------- - job_id : uuid - id of the job - job_status : JobStatusType - current status of the job (eg. notstarted, succeeded, failed) - exec_profile : ExecProfile - python exec profile - inputs : dict - input parameters of the job - py_exec_session : PyExecSession - session in which the job will be run - """ - Job.__init__(self, job_id, job_status, exec_profile) - self.inputs = inputs - self.wf_lang = exec_profile.wf_lang - - def prepare(self) -> None: - """ - Prepares the job. - - Parameters - ---------- - - Returns - ------- - NONE - """ - - def exec(self) -> None: - """ - Executes the job. - - Parameters - ---------- - - Returns - ------- - NONE - """ - - def eval(self) -> None: - """ - Evaluates the job. - - Parameters - ---------- - - Returns - ------- - NONE - """ - - def finalize(self) -> None: - """ - Finalizes the job. - - Parameters - ---------- - - Returns - ------- - NONE - """ - - def cancel(self) -> None: - """ - Cancels the job. - - Parameters - ---------- - - Returns - ------- - NONE - """ diff --git a/exec_manager/wf_lang_type.py b/exec_manager/utils.py similarity index 76% rename from exec_manager/wf_lang_type.py rename to exec_manager/utils.py index 6fa7774..2fa4a16 100644 --- a/exec_manager/wf_lang_type.py +++ b/exec_manager/utils.py @@ -12,22 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""enum for workflow languagae type""" +"""module for utility stuff""" from enum import Enum class WfLangType(Enum): - """enumerate workflow language types""" + """Enumerate workflow language types: + - CWL: cwl language + - WDL: wdl language + - NEXTFLOW: nextflow language + - SNAKEMAKE: snakemake language + """ CWL = "cwl" - """cwl language""" - WDL = "wdl" - """wdl language""" - NEXTFLOW = "nextflow" - """nextflow language""" - SNAKEMAKE = "snakemake" - """snakemake language""" + + +class hello: + print("test") diff --git a/setup.cfg b/setup.cfg index f0786f3..07d04da 100644 --- a/setup.cfg +++ b/setup.cfg @@ -72,6 +72,7 @@ dev = typer==0.4.1 sqlalchemy-utils==0.38.2 sqlalchemy-stubs==0.4 + pydantic==1.9.1 # Please adapt: Only needed if you are using alembic for database versioning (Probably for PostgreSQL) db_migration = diff --git a/tests/unit/test_job_dao.py b/tests/unit/test_job_dao.py index 8288d16..e638783 100644 --- a/tests/unit/test_job_dao.py +++ b/tests/unit/test_job_dao.py @@ -14,104 +14,80 @@ import json -from exec_manager.dao.job_dao import create_job_dao, get_job, update_job_status -from exec_manager.exec_profile import ExecProfile -from exec_manager.exec_profile_type import ExecProfileType -from exec_manager.job_status_type import JobStatusType -from exec_manager.wf_lang_type import WfLangType - -# import pytest -# from exec_manager.dao.db_models import metadata -# from sqlalchemy import create_engine - - -# @pytest.fixture -# def example_engine(): -# engine = create_engine("sqlite+pysqlite://") -# metadata.create_all(engine) -# return engine - - -# @pytest.fixture -# def example_job_id(): -# return create_job_dao( -# JobStatusType.NOTSTARTET, -# ExecProfile(ExecProfileType.PYTHON, WfLangType.CWL), -# json.dumps({"test": 1}), -# {"hello": "world"}, -# ) +import pytest +from exec_manager.dao.job_dao import SQLJobDAO +from exec_manager.exec_profiles import ExecProfile, ExecProfileType +from exec_manager.jobs import JobStatusType, PythonJob +from exec_manager.utils import WfLangType -# engine = create_engine("sqlite+pysqlite://") -# metadata.create_all(engine) -job_status = JobStatusType.NOTSTARTET -exec_profile = ExecProfile(ExecProfileType.PYTHON, WfLangType.CWL) -workflow = {"test": 1} -inputs = {"hello": "world"} -example_job_id = create_job_dao( - JobStatusType.NOTSTARTET, - ExecProfile(ExecProfileType.PYTHON, WfLangType.CWL), - workflow, - inputs, -) +@pytest.fixture +def example_sqlite_job_dao(): + engine = SQLJobDAO("sqlite+pysqlite://") + yield engine -# @pytest.mark.usefixtures("example_engine") -def test_create_job_dao(): - job_id = create_job_dao(job_status, exec_profile, workflow, inputs) - db_job = get_job(job_id) - assert ( - str(db_job.job_id) == str(job_id) - and db_job.job_status.value == job_status.value - and ( - json.dumps( - { - "exec_profile_type": db_job.exec_profile.exec_profile_type.value, - "wf_lang": db_job.exec_profile.wf_lang.value, - } - ) - == json.dumps( - { - "exec_profile_type": ExecProfileType.PYTHON.value, - "wf_lang": WfLangType.CWL.value, - } - ) - ) - ) +@pytest.fixture +def example_job_status(): + return JobStatusType.NOTSTARTET -# @pytest.mark.usefixtures("example_engine") -def test_get_job(): - job_id = example_job_id - db_job = get_job(job_id) - assert ( - str(db_job.job_id) == str(job_id) - and db_job.job_status.value == job_status.value - and ( - json.dumps( - { - "exec_profile_type": db_job.exec_profile.exec_profile_type.value, - "wf_lang": db_job.exec_profile.wf_lang.value, - } - ) - == json.dumps( - { - "exec_profile_type": ExecProfileType.PYTHON.value, - "wf_lang": WfLangType.CWL.value, - } - ) - ) + +@pytest.fixture +def example_exec_profile(): + return ExecProfile(ExecProfileType.PYTHON, WfLangType.CWL) + + +@pytest.fixture +def example_workflow(): + return {"test": 1} + + +@pytest.fixture +def example_inputs(): + return {"hello": "world"} + + +@pytest.fixture +def example_job_id( + example_job_status, + example_exec_profile, + example_workflow, + example_inputs, + example_sqlite_job_dao, +): + return example_sqlite_job_dao.create( + example_job_status, + example_exec_profile, + example_workflow, + example_inputs, ) -# @pytest.mark.usefixtures("example_engine") -def test_update_job_status(): - job_id = example_job_id - update_job_status(job_id, JobStatusType.PREPARING) - db_job = get_job(job_id) +@pytest.mark.usefixtures( + "example_sqlite_job_dao", + "example_job_id", + "example_exec_profile", + "example_workflow", + "example_inputs", +) +def test_create( + example_job_status, + example_exec_profile, + example_workflow, + example_inputs, + example_sqlite_job_dao, +): + job_id = example_sqlite_job_dao.create( + example_job_status, + example_exec_profile, + example_workflow, + example_inputs, + ) + db_job = example_sqlite_job_dao.get(job_id) assert ( str(db_job.job_id) == str(job_id) - and db_job.job_status.value == JobStatusType.PREPARING.value + and db_job.job_status.value == example_job_status.value and ( json.dumps( { @@ -121,9 +97,65 @@ def test_update_job_status(): ) == json.dumps( { - "exec_profile_type": ExecProfileType.PYTHON.value, - "wf_lang": WfLangType.CWL.value, + "exec_profile_type": example_exec_profile.exec_profile_type.value, + "wf_lang": example_exec_profile.wf_lang.value, } ) ) ) + + +# @pytest.mark.usefixtures( +# "example_sqlite_job_dao", "example_job_id", "example_exec_profile" +# ) +# def test_get( +# example_job_id, example_job_status, example_exec_profile, example_sqlite_job_dao +# ): +# job_id = example_job_id +# db_job = example_sqlite_job_dao.get(job_id) +# assert ( +# str(db_job.job_id) == str(job_id) +# and db_job.job_status.value == example_job_status.value +# and ( +# json.dumps( +# { +# "exec_profile_type": db_job.exec_profile.exec_profile_type.value, +# "wf_lang": db_job.exec_profile.wf_lang.value, +# } +# ) +# == json.dumps( +# { +# "exec_profile_type": example_exec_profile.exec_profile_type.value, +# "wf_lang": example_exec_profile.wf_lang.value, +# } +# ) +# ) +# ) + + +# @pytest.mark.usefixtures( +# "example_sqlite_job_dao", "example_job_id", "example_exec_profile" +# ) +# def test_update( +# example_job_id, example_exec_profile, example_inputs, example_sqlite_job_dao +# ): +# job_id = example_job_id +# job = PythonJob( +# job_id, JobStatusType.PREPARING, example_exec_profile, example_inputs +# ) +# example_sqlite_job_dao.update(job_id, job) +# db_job = example_sqlite_job_dao.get(job_id) +# assert ( +# str(db_job.job_id) == str(job_id) +# and db_job.job_status.value == JobStatusType.PREPARING.value +# and ( +# { +# "exec_profile_type": db_job.exec_profile.exec_profile_type.value, +# "wf_lang": db_job.exec_profile.wf_lang.value, +# } +# == { +# "exec_profile_type": example_exec_profile.exec_profile_type.value, +# "wf_lang": example_exec_profile.wf_lang.value, +# } +# ) +# ) From e5e559bef87a213de911f2b8782aea3aa0592e10 Mon Sep 17 00:00:00 2001 From: e-buerger Date: Mon, 23 May 2022 09:15:12 +0200 Subject: [PATCH 31/33] remove unneccassery class --- exec_manager/utils.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/exec_manager/utils.py b/exec_manager/utils.py index 2fa4a16..3293609 100644 --- a/exec_manager/utils.py +++ b/exec_manager/utils.py @@ -29,7 +29,3 @@ class WfLangType(Enum): WDL = "wdl" NEXTFLOW = "nextflow" SNAKEMAKE = "snakemake" - - -class hello: - print("test") From 86b72dda581879c20ef4a14eb6fb4fe29f3a01c8 Mon Sep 17 00:00:00 2001 From: e-buerger Date: Mon, 23 May 2022 08:30:26 +0000 Subject: [PATCH 32/33] satisfy pipeline --- exec_manager/dao/job_dao.py | 6 +- exec_manager/job_execution.py | 129 ++++++++++++++++ exec_manager/jobs.py | 110 +------------- tests/unit/test_job_dao.py | 278 +++++++++++++++++++++++++--------- 4 files changed, 339 insertions(+), 184 deletions(-) create mode 100644 exec_manager/job_execution.py diff --git a/exec_manager/dao/job_dao.py b/exec_manager/dao/job_dao.py index f7ee3b3..c56b67f 100644 --- a/exec_manager/dao/job_dao.py +++ b/exec_manager/dao/job_dao.py @@ -162,13 +162,13 @@ def get(self, job_id: UUID) -> Job: job_status = JobStatusType(result[0][1]) exec_profile = json.loads(result[0][2]) exec_profile = ExecProfile( - ExecProfileType(exec_profile["exec_profile_type"]), - WfLangType(exec_profile["wf_lang"]), + type_=ExecProfileType(exec_profile["exec_profile_type"]), + wf_lang=WfLangType(exec_profile["wf_lang"]), ) inputs = json.loads(result[0][4]) if exec_profile.type_ == ExecProfileType.PYTHON: return PythonJob(job_id, job_status, exec_profile, inputs) - if exec_profile.exec_profile_type == ExecProfileType.BASH: + if exec_profile.type_ == ExecProfileType.BASH: raise NotImplementedError( "Execution profiles of type Bash not supported, yet" ) diff --git a/exec_manager/job_execution.py b/exec_manager/job_execution.py new file mode 100644 index 0000000..f37f1a9 --- /dev/null +++ b/exec_manager/job_execution.py @@ -0,0 +1,129 @@ +# Copyright 2021 - 2022 German Cancer Research Center (DKFZ) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""module for job execution""" + +from typing import Callable +from uuid import UUID + +from exec_manager.dao.job_dao import SQLJobDAO +from exec_manager.exec_profiles import ExecProfile, ExecProfileType +from exec_manager.jobs import Job, JobStatusType, PythonJob + + +class PyExecSession: + """ + class for python job + + ... + + Attributes + ---------- + max_retries : int + + Methods + ------- + run() -> None + runs a job + """ + + def __init__( + self, + max_retries: int = 0, + ) -> None: + """ + Constructs all the necessary attributes for the python exec session. + + Parameters + ---------- + max_retries : int + number of maximum retries when the execution fails (default: 0) + """ + self._max_retries = max_retries + + def run( + self, + job_id: UUID, + job_status: JobStatusType, + exec_profile: ExecProfile, + inputs: dict, + ) -> None: + """ + runs a job. + + Parameters + ---------- + job_id: UUID + id of the job + job_status: JobStatusType + current status of the job (e.g. notstarted, executing, failed, ...) + exec_profile: ExecProfile + exec profile with which the job should be executed (bash, python, WES) + inputs: dict, + + Returns + ------- + NONE + """ + counter = -1 + sql_job_dao = SQLJobDAO("sqlite+pysqlite://") + while self._max_retries > counter: + python_job = PythonJob(job_id, job_status, exec_profile, inputs) + python_job.job_status = JobStatusType.PREPARING + sql_job_dao.update(job_id, python_job) + python_job.prepare() + python_job.job_status = JobStatusType.EXECUTING + sql_job_dao.update(job_id, python_job) + python_job.exec() + python_job.job_status = JobStatusType.EVALUATING + sql_job_dao.update(job_id, python_job) + python_job.eval() + python_job.job_status = JobStatusType.FINALZING + sql_job_dao.update(job_id, python_job) + python_job.finalize() + if sql_job_dao.get(job_id).job_status == JobStatusType.SUCCEEDED: + break + counter = counter + 1 + + +def create_job( + inputs: dict, + workflow: dict, + exec_profile: ExecProfile, + create: Callable = SQLJobDAO.create, +) -> Job: + """ + Creates a job. + + Parameters + ---------- + inputs: dict + input paramters of the job + workflow: dict + the job's workflow + exec_profile: ExecProfile + exec profile with which the job should be executed (bash, python, WES) + + Returns + ------- + Job + """ + job_status = JobStatusType.NOTSTARTET + job_id = create(job_status, exec_profile, workflow, inputs) + if exec_profile.type_ == ExecProfileType.PYTHON: + return PythonJob(job_id, job_status, exec_profile, inputs) + if exec_profile.type_ == ExecProfileType.BASH: + raise NotImplementedError("Execution profiles of type Bash not supported, yet") + raise NotImplementedError("Execution profiles of type WES not supported, yet") diff --git a/exec_manager/jobs.py b/exec_manager/jobs.py index 06b2426..0d1bca2 100644 --- a/exec_manager/jobs.py +++ b/exec_manager/jobs.py @@ -16,11 +16,9 @@ from abc import ABC, abstractmethod from enum import Enum -from typing import Callable from uuid import UUID -from exec_manager.dao.job_dao import SQLJobDAO -from exec_manager.exec_profiles import ExecProfile, ExecProfileType +from exec_manager.exec_profiles import ExecProfile class JobStatusType(Enum): @@ -279,109 +277,3 @@ def cancel(self) -> None: ------- NONE """ - - -class PyExecSession: - """ - class for python job - - ... - - Attributes - ---------- - max_retries : int - - Methods - ------- - run() -> None - runs a job - """ - - def __init__( - self, - max_retries: int = 0, - ) -> None: - """ - Constructs all the necessary attributes for the python exec session. - - Parameters - ---------- - max_retries : int - number of maximum retries when the execution fails (default: 0) - """ - self._max_retries = max_retries - - def run( - self, - job_id: UUID, - job_status: JobStatusType, - exec_profile: ExecProfile, - inputs: dict, - ) -> None: - """ - runs a job. - - Parameters - ---------- - job_id: UUID - id of the job - job_status: JobStatusType - current status of the job (e.g. notstarted, executing, failed, ...) - exec_profile: ExecProfile - exec profile with which the job should be executed (bash, python, WES) - inputs: dict, - - Returns - ------- - NONE - """ - counter = -1 - sql_job_dao = SQLJobDAO("sqlite+pysqlite://") - while self._max_retries > counter: - python_job = PythonJob(job_id, job_status, exec_profile, inputs) - python_job.job_status = JobStatusType.PREPARING - sql_job_dao.update(job_id, python_job) - python_job.prepare() - python_job.job_status = JobStatusType.EXECUTING - sql_job_dao.update(job_id, python_job) - python_job.exec() - python_job.job_status = JobStatusType.EVALUATING - sql_job_dao.update(job_id, python_job) - python_job.eval() - python_job.job_status = JobStatusType.FINALZING - sql_job_dao.update(job_id, python_job) - python_job.finalize() - if sql_job_dao.get(job_id).job_status == JobStatusType.SUCCEEDED: - break - counter = counter + 1 - - -def create_job( - inputs: dict, - workflow: dict, - exec_profile: ExecProfile, - create: Callable = SQLJobDAO.create, -) -> Job: - """ - Creates a job. - - Parameters - ---------- - inputs: dict - input paramters of the job - workflow: dict - the job's workflow - exec_profile: ExecProfile - exec profile with which the job should be executed (bash, python, WES) - - Returns - ------- - Job - """ - job_status = JobStatusType.NOTSTARTET - job_id = create(job_status, exec_profile, workflow, inputs) - if exec_profile.exec_profile_type == ExecProfileType.PYTHON: - return PythonJob(job_id, job_status, exec_profile, inputs) - if exec_profile.exec_profile_type == ExecProfileType.BASH: - raise NotImplementedError("Execution profiles of type Bash not supported, yet") - raise NotImplementedError("Execution profiles of type WES not supported, yet") diff --git a/tests/unit/test_job_dao.py b/tests/unit/test_job_dao.py index e638783..a969427 100644 --- a/tests/unit/test_job_dao.py +++ b/tests/unit/test_job_dao.py @@ -13,6 +13,8 @@ # limitations under the License. import json +from typing import Generator +from uuid import UUID import pytest @@ -23,39 +25,109 @@ @pytest.fixture -def example_sqlite_job_dao(): +def example_sqlite_job_dao() -> Generator: + """ + Creates sqlite engine. + + Parameters + ---------- + + Returns + ------- + None + """ engine = SQLJobDAO("sqlite+pysqlite://") yield engine @pytest.fixture -def example_job_status(): +def example_job_status() -> JobStatusType: + """ + Creates a job status. + + Parameters + ---------- + + Returns + ------- + JobStatusType + """ return JobStatusType.NOTSTARTET @pytest.fixture -def example_exec_profile(): - return ExecProfile(ExecProfileType.PYTHON, WfLangType.CWL) +def example_exec_profile() -> ExecProfile: + """ + Creates an exec profile. + + Parameters + ---------- + + Returns + ------- + ExecProfile + """ + return ExecProfile(type_=ExecProfileType.PYTHON, wf_lang=WfLangType.CWL) @pytest.fixture -def example_workflow(): +def example_workflow() -> dict: + """ + Creates a workflow. + + Parameters + ---------- + + Returns + ------- + dict + """ return {"test": 1} @pytest.fixture -def example_inputs(): +def example_inputs() -> dict: + """ + Creates inputs. + + Parameters + ---------- + + Returns + ------- + dict + """ return {"hello": "world"} @pytest.fixture def example_job_id( - example_job_status, - example_exec_profile, - example_workflow, - example_inputs, + example_job_status: JobStatusType, + example_exec_profile: ExecProfile, + example_workflow: dict, + example_inputs: dict, example_sqlite_job_dao, -): +) -> UUID: + """ + Creates a job id. + + Parameters + ---------- + example_job_status: JobStatusType + status of the job + example_exec_profile: ExecProfile + execution profile of the job + example_workflow: dict + job's workflow + example_inputs: dict + input parameters of the workflow + example_sqlite_job_dao + database engine + + Returns + ------- + UUID + """ return example_sqlite_job_dao.create( example_job_status, example_exec_profile, @@ -72,12 +144,32 @@ def example_job_id( "example_inputs", ) def test_create( - example_job_status, - example_exec_profile, - example_workflow, - example_inputs, + example_job_status: JobStatusType, + example_exec_profile: ExecProfile, + example_workflow: dict, + example_inputs: dict, example_sqlite_job_dao, -): +) -> None: + """ + Tests the create method from SQLJobDAO class. + + Parameters + ---------- + example_job_status: JobStatusType + status of the job + example_exec_profile: ExecProfile + execution profile of the job + example_workflow: dict + job's workflow + example_inputs: dict + input parameters of the workflow + example_sqlite_job_dao + database engine + + Returns + ------- + None + """ job_id = example_sqlite_job_dao.create( example_job_status, example_exec_profile, @@ -91,13 +183,62 @@ def test_create( and ( json.dumps( { - "exec_profile_type": db_job.exec_profile.exec_profile_type.value, + "exec_profile_type": db_job.exec_profile.type_.value, + "wf_lang": db_job.exec_profile.wf_lang.value, + } + ) + == json.dumps( + { + "exec_profile_type": example_exec_profile.type_.value, + "wf_lang": example_exec_profile.wf_lang.value, + } + ) + ) + ) + + +@pytest.mark.usefixtures( + "example_sqlite_job_dao", "example_job_id", "example_exec_profile" +) +def test_get( + example_job_id: UUID, + example_job_status: JobStatusType, + example_exec_profile: ExecProfile, + example_sqlite_job_dao, +) -> None: + """ + Tests the get method from SQLJobDAO class. + + Parameters + ---------- + example_job_id: UUID + id of the job + example_job_status: JobStatusType + status of the job + example_exec_profile: ExecProfile + execution profile of the job + example_sqlite_job_dao + database engine + + Returns + ------- + None + """ + job_id = example_job_id + db_job = example_sqlite_job_dao.get(job_id) + assert ( + str(db_job.job_id) == str(job_id) + and db_job.job_status.value == example_job_status.value + and ( + json.dumps( + { + "exec_profile_type": db_job.exec_profile.type_.value, "wf_lang": db_job.exec_profile.wf_lang.value, } ) == json.dumps( { - "exec_profile_type": example_exec_profile.exec_profile_type.value, + "exec_profile_type": example_exec_profile.type_.value, "wf_lang": example_exec_profile.wf_lang.value, } ) @@ -105,57 +246,50 @@ def test_create( ) -# @pytest.mark.usefixtures( -# "example_sqlite_job_dao", "example_job_id", "example_exec_profile" -# ) -# def test_get( -# example_job_id, example_job_status, example_exec_profile, example_sqlite_job_dao -# ): -# job_id = example_job_id -# db_job = example_sqlite_job_dao.get(job_id) -# assert ( -# str(db_job.job_id) == str(job_id) -# and db_job.job_status.value == example_job_status.value -# and ( -# json.dumps( -# { -# "exec_profile_type": db_job.exec_profile.exec_profile_type.value, -# "wf_lang": db_job.exec_profile.wf_lang.value, -# } -# ) -# == json.dumps( -# { -# "exec_profile_type": example_exec_profile.exec_profile_type.value, -# "wf_lang": example_exec_profile.wf_lang.value, -# } -# ) -# ) -# ) - - -# @pytest.mark.usefixtures( -# "example_sqlite_job_dao", "example_job_id", "example_exec_profile" -# ) -# def test_update( -# example_job_id, example_exec_profile, example_inputs, example_sqlite_job_dao -# ): -# job_id = example_job_id -# job = PythonJob( -# job_id, JobStatusType.PREPARING, example_exec_profile, example_inputs -# ) -# example_sqlite_job_dao.update(job_id, job) -# db_job = example_sqlite_job_dao.get(job_id) -# assert ( -# str(db_job.job_id) == str(job_id) -# and db_job.job_status.value == JobStatusType.PREPARING.value -# and ( -# { -# "exec_profile_type": db_job.exec_profile.exec_profile_type.value, -# "wf_lang": db_job.exec_profile.wf_lang.value, -# } -# == { -# "exec_profile_type": example_exec_profile.exec_profile_type.value, -# "wf_lang": example_exec_profile.wf_lang.value, -# } -# ) -# ) +@pytest.mark.usefixtures( + "example_sqlite_job_dao", "example_job_id", "example_exec_profile" +) +def test_update( + example_job_id: UUID, + example_exec_profile: ExecProfile, + example_inputs: dict, + example_sqlite_job_dao, +) -> None: + """ + Tests the update method from SQLJobDAO class. + + Parameters + ---------- + example_job_id: UUID + id of the job + example_exec_profile: ExecProfile + execution profile of the job + example_inputs: dict + input parameters of the workflow + example_sqlite_job_dao: Engine + database engine + + Returns + ------- + None + """ + job_id = example_job_id + job = PythonJob( + job_id, JobStatusType.PREPARING, example_exec_profile, example_inputs + ) + example_sqlite_job_dao.update(job_id, job) + db_job = example_sqlite_job_dao.get(job_id) + assert ( + str(db_job.job_id) == str(job_id) + and db_job.job_status.value == JobStatusType.PREPARING.value + and ( + { + "exec_profile_type": db_job.exec_profile.type_.value, + "wf_lang": db_job.exec_profile.wf_lang.value, + } + == { + "exec_profile_type": example_exec_profile.type_.value, + "wf_lang": example_exec_profile.wf_lang.value, + } + ) + ) From 985661004de4ae252c6dffeed9ccf1f8197f9229 Mon Sep 17 00:00:00 2001 From: e-buerger Date: Mon, 23 May 2022 13:41:30 +0000 Subject: [PATCH 33/33] update doc strings --- exec_manager/dao/db_models.py | 11 +- exec_manager/dao/job_dao.py | 165 ++++++++++++------------- exec_manager/exec_profiles.py | 17 +-- exec_manager/job_execution.py | 81 +++++-------- exec_manager/jobs.py | 222 ++++++++-------------------------- tests/unit/test_job_dao.py | 152 +++++++---------------- 6 files changed, 216 insertions(+), 432 deletions(-) diff --git a/exec_manager/dao/db_models.py b/exec_manager/dao/db_models.py index be125fa..e504f60 100644 --- a/exec_manager/dao/db_models.py +++ b/exec_manager/dao/db_models.py @@ -23,13 +23,22 @@ Base: DeclarativeMeta = declarative_base() metadata = Base.metadata +# this method is neccessary to avoid IntegrityError +def generate_uuid_str() -> str: + """Generates a uuid with type string. + + Returns: + str: job id + """ + return str(uuid.uuid4()) + class DBJob(Base): """An job object stored in the DB""" __tablename__ = "job" - job_id = Column(String, default=uuid.uuid4(), primary_key=True) + job_id = Column(String, default=generate_uuid_str, primary_key=True) job_status = Column(String, nullable=False) exec_profile = Column(JSON, nullable=False) workflow = Column(JSON, nullable=False) diff --git a/exec_manager/dao/job_dao.py b/exec_manager/dao/job_dao.py index c56b67f..85b8ee1 100644 --- a/exec_manager/dao/job_dao.py +++ b/exec_manager/dao/job_dao.py @@ -14,7 +14,6 @@ """module for job dao""" -import json from abc import ABC, abstractmethod from uuid import UUID @@ -27,7 +26,13 @@ class JobDAO(ABC): - """abstract class for job dao""" + """abstract class for job dao + + Methods: + create (UUID): inserts a job into database + get (Job): returns a job by its id + update (): updates a job by its id + """ @abstractmethod def create( @@ -37,63 +42,51 @@ def create( workflow: dict, inputs: dict, ) -> UUID: - """ - Inserts a job into the database. - - Parameters - ---------- - job_status: JobStatusType - current status of the job; initially it is JobStatusType.NOTSTARTED - exec_profile: ExecProfile - exec profile of this job - workflow - the jobs workflow - inputs: dict - the input parameters of the job - - Returns - ------- - UUID + """Inserts a job into the database. + + Args: + job_status (JobStatusType): current status of the job; + initially it is JobStatusType.NOTSTARTED + exec_profile (ExecProfile): exec profile of this job + workflow (dict): the jobs workflow + inputs (dict): the input parameters of the job + + Returns: + UUID: job id """ ... @abstractmethod def get(self, job_id: UUID) -> Job: - """ - Returns a job by his job id. + """Returns a job by its job id. - Parameters - ---------- - job_id: UUID - id of the job + Args: + job_id (UUID): id of the job - Returns - ------- - Job + Returns: + Job: job belongig to job id """ ... @abstractmethod def update(self, job_id: UUID, job: Job) -> None: - """ - Updates a jobs by his id. - - Parameters - ---------- - job_id: UUID - id of the job - job: Job - updated job - - Returns - ------- - None + """Updates a jobs by its id. + + Args: + job_id (UUID): id of the job + job (Job): updated job """ ... class SQLJobDAO(JobDAO): - """class for sql job dao""" + """class for sql job dao + + Methods: + create (UUID): inserts a job into database + get (Job): returns a job by its id + update (): updates a job by its id + """ def __init__(self, db_url: str): """Initialize DB.""" @@ -107,50 +100,45 @@ def create( workflow: dict, inputs: dict, ) -> UUID: - """ - Inserts a job into the database. - - Parameters - ---------- - job_status: JobStatusType - current status of the job; initially it is JobStatusType.NOTSTARTED - exec_profile: ExecProfile - exec profile of this job - workflow: dict - the jobs workflow - inputs: dict - the input parameters of the job - - Returns - ------- - UUID + """Inserts a job into the database. + + Args: + job_status (JobStatusType): current status of the job; + initially it is JobStatusType.NOTSTARTED + exec_profile (ExecProfile): exec profile of this job + workflow (dict): the jobs workflow + inputs (dict): the input parameters of the job + + Returns: + UUID: job id """ with self._engine.connect() as connection: cursor = connection.execute( - insert(DBJob.__table__) - .values( + insert(DBJob.__table__).values( job_status=job_status.value, - exec_profile=exec_profile.dict(), + exec_profile={ + "exec_profile_type": exec_profile.type_.value, + "wf_lang_type": exec_profile.wf_lang.value, + }, workflow=workflow, inputs=inputs, ) - .returning(DBJob.job_id) ) - result = cursor.fetchall - return result[0][0] # job_id + job_id = cursor.inserted_primary_key[0] + return job_id def get(self, job_id: UUID) -> Job: - """ - Returns a job by his job id. + """Returns a job by its job id. + + Args: + job_id (UUID): id of the job - Parameters - ---------- - job_id: UUID - id of the job + Raises: + NotImplementedError: Bash exec profile is not implemented yet + NotImplementedError: WES exec profile is not implemented yet - Returns - ------- - Job + Returns: + Job: job belonging to job_id """ with self._engine.connect() as connection: cursor = connection.execute( @@ -160,13 +148,13 @@ def get(self, job_id: UUID) -> Job: ) result = cursor.fetchall() job_status = JobStatusType(result[0][1]) - exec_profile = json.loads(result[0][2]) + exec_profile = result[0][2] exec_profile = ExecProfile( type_=ExecProfileType(exec_profile["exec_profile_type"]), - wf_lang=WfLangType(exec_profile["wf_lang"]), + wf_lang=WfLangType(exec_profile["wf_lang_type"]), ) - inputs = json.loads(result[0][4]) if exec_profile.type_ == ExecProfileType.PYTHON: + inputs = result[0][4] return PythonJob(job_id, job_status, exec_profile, inputs) if exec_profile.type_ == ExecProfileType.BASH: raise NotImplementedError( @@ -177,19 +165,11 @@ def get(self, job_id: UUID) -> Job: ) def update(self, job_id: UUID, job: Job) -> None: - """ - Updates a jobs by his id. - - Parameters - ---------- - job_id: UUID - id of the job - job: Job - updated job - - Returns - ------- - None + """Updates a jobs by its id. + + Args: + job_id (UUID): id of the job + job (Job): updated job """ with self._engine.connect() as connection: connection.execute( @@ -197,6 +177,9 @@ def update(self, job_id: UUID, job: Job) -> None: .where(DBJob.job_id == str(job_id)) .values( job_status=job.job_status.value, - exec_profile=job.exec_profile.dict(), + exec_profile={ + "exec_profile_type": job.exec_profile.type_.value, + "wf_lang_type": job.exec_profile.wf_lang.value, + }, ) ) diff --git a/exec_manager/exec_profiles.py b/exec_manager/exec_profiles.py index 20aa9f5..8223ef8 100644 --- a/exec_manager/exec_profiles.py +++ b/exec_manager/exec_profiles.py @@ -34,20 +34,11 @@ class ExecProfileType(Enum): class ExecProfile(BaseModel): - """ - class for Exec-Profile - - ... - - Attributes - ---------- - exec_profile_type : ExecProfileType - type of the exec profile (bash, python, wes) - wf_lang : WfLangType - workflow language (cwl, wdl, nextflow, snakemake) + """Base model class for execution profiles. - Methods - ------- + Args: + type_ (ExecProfileType): type of the execution profile (Python, Bash or WES) + wf_lang (WfLangType): language type of workflow (CWL, WDL, Snakemake, Nextflow) """ type_: ExecProfileType diff --git a/exec_manager/job_execution.py b/exec_manager/job_execution.py index f37f1a9..3c7132a 100644 --- a/exec_manager/job_execution.py +++ b/exec_manager/job_execution.py @@ -24,32 +24,21 @@ class PyExecSession: - """ - class for python job - - ... - - Attributes - ---------- - max_retries : int + """Class for PyExecSession - Methods - ------- - run() -> None - runs a job + Methods: + run (): runs a job """ def __init__( self, max_retries: int = 0, ) -> None: - """ - Constructs all the necessary attributes for the python exec session. + """Constructs all the necessary attributes for the python exec session. - Parameters - ---------- - max_retries : int - number of maximum retries when the execution fails (default: 0) + Args: + max_retries (int, optional): number of maximum retries when the execution + fails. Defaults to 0. """ self._max_retries = max_retries @@ -60,22 +49,15 @@ def run( exec_profile: ExecProfile, inputs: dict, ) -> None: - """ - runs a job. - - Parameters - ---------- - job_id: UUID - id of the job - job_status: JobStatusType - current status of the job (e.g. notstarted, executing, failed, ...) - exec_profile: ExecProfile - exec profile with which the job should be executed (bash, python, WES) - inputs: dict, - - Returns - ------- - NONE + """Runs a job. + + Args: + job_id (UUID): id of the job + job_status (JobStatusType): current status of the job + (e.g. notstarted, executing, failed, ...) + exec_profile (ExecProfile): exec profile with which the job should be + executed (bash, python, WES) + inputs (dict): input parameters of the workflow """ counter = -1 sql_job_dao = SQLJobDAO("sqlite+pysqlite://") @@ -104,21 +86,22 @@ def create_job( exec_profile: ExecProfile, create: Callable = SQLJobDAO.create, ) -> Job: - """ - Creates a job. - - Parameters - ---------- - inputs: dict - input paramters of the job - workflow: dict - the job's workflow - exec_profile: ExecProfile - exec profile with which the job should be executed (bash, python, WES) - - Returns - ------- - Job + """Creates a job. + + Args: + inputs (dict): input paramters of the job + workflow (dict): the job's workflow + exec_profile (ExecProfile): exec profile with which the job should be executed + (bash, python, WES) + create (Callable, optional): function that stores the job in a database. + Defaults to SQLJobDAO.create. + + Raises: + NotImplementedError: Bash execution profile not implemented yet + NotImplementedError: WES execution profile not implemented yet + + Returns: + Job: created job """ job_status = JobStatusType.NOTSTARTET job_id = create(job_status, exec_profile, workflow, inputs) diff --git a/exec_manager/jobs.py b/exec_manager/jobs.py index 0d1bca2..0e4a2c9 100644 --- a/exec_manager/jobs.py +++ b/exec_manager/jobs.py @@ -44,48 +44,33 @@ class JobStatusType(Enum): class Job(ABC): - """ - class for job + """abstract class for job - ... + Args: + job_id (UUID): id of the job + job_status (JobStatusType): current status of the job (eg. notstarted, succeeded, failed) + exec_profile (ExecProfile): exec profile with which the job should be executed - Attributes - ---------- - job_id : uuid - id of the job - job_status : JobStatusType - current status of the job (eg. notstarted, succeeded, failed) - exec_profile : ExecProfile - exec profile with which the job should be executed + Methods: + prepare (): prepares the job + exec (): executes the job + eval (): evaluates the job + finalize (): finalizes the job + cancel (): cancels the job - Methods - ------- - prepare() -> None: - prepares the job - exec() -> None: - executes the job - eval() -> None: - evaluates the job - finalize() -> None: - finalizes the job - cancel() -> None: - cancels the job """ def __init__( self, job_id: UUID, job_status: JobStatusType, exec_profile: ExecProfile ) -> None: - """ - Constructs all the necessary attributes for the job object. + """Constructs all the necessary attributes for the job object. - Parameters - ---------- - job_id : UUID - id of the job - job_status : JobStatusType - current status of the job (eg. notstarted, succeeded, failed, ...) - exec_profile : ExecProfile - exec profile with which the job should be executed (bash, python, WES) + Args: + job_id (UUID): id of the job + job_status (JobStatusType): current status of the job + (eg. notstarted, succeeded, failed, ...) + exec_profile (ExecProfile): exec profile with which the job should be + executed (bash, python, WES) """ self.job_id = job_id self.job_status = job_status @@ -93,72 +78,27 @@ def __init__( @abstractmethod def prepare(self) -> None: - """ - Prepares the job. - - Parameters - ---------- - - Returns - ------- - NONE - """ + """Prepares the job.""" ... @abstractmethod def exec(self) -> None: - """ - Executes the job. - - Parameters - ---------- - - Returns - ------- - NONE - """ + """Executes the job.""" ... @abstractmethod def eval(self) -> None: - """ - Evaluates the job. - - Parameters - ---------- - - Returns - ------- - NONE - """ + """Evaluates the job.""" ... @abstractmethod def finalize(self) -> None: - """ - Finalizes the job. - - Parameters - ---------- - - Returns - ------- - NONE - """ + """Finalizes the job.""" ... @abstractmethod def cancel(self) -> None: - """ - Cancels the job. - - Parameters - ---------- - - Returns - ------- - NONE - """ + """Cancels the job.""" ... @@ -166,31 +106,18 @@ class PythonJob(Job): """ class for python job - ... - - Attributes - ---------- - job_id : UUID - id of the job - job_status : JobStatusType - current status of the job (eg. notstarted, succeeded, failed) - exec_profile : ExecProfile - python exec profile - inputs : dict - input parameters of the job - - Methods - ------- - prepare() -> None: - prepares the job - exec() -> None: - executes the job - eval() -> None: - evaluates the job - finalize() -> None: - finalizes the job - cancel() -> None: - cancels the job + Args: + job_id (UUID): id of the job + job_status (JobStatusType): current status of the job (eg. notstarted, succeeded, failed) + exec_profile (ExecProfile): python exec profile + inputs (dict): input parameters of the job + + Methods: + prepare (): prepares the job + exec (): executes the job + eval (): evaluates the job + finalize (): finalizes the job + cancel (): cancels the job """ def __init__( @@ -200,80 +127,35 @@ def __init__( exec_profile: ExecProfile, inputs: dict, ) -> None: - """ - Constructs all the necessary attributes for the python job object. + """Constructs all the necessary attributes for the python job object. - Parameters - ---------- - job_id : uuid - id of the job - job_status : JobStatusType - current status of the job (eg. notstarted, succeeded, failed) - exec_profile : ExecProfile - python exec profile - inputs : dict - input parameters of the job + Args: + job_id (UUID): id of the job + job_status (JobStatusType): current status of the job + (eg. notstarted, succeeded, failed) + exec_profile (ExecProfile): python exec profile + inputs (dict): input parameters of the job """ Job.__init__(self, job_id, job_status, exec_profile) self.inputs = inputs self.wf_lang = exec_profile.wf_lang def prepare(self) -> None: - """ - Prepares the job. - - Parameters - ---------- - - Returns - ------- - NONE - """ + """Prepares the job.""" + ... def exec(self) -> None: - """ - Executes the job. - - Parameters - ---------- - - Returns - ------- - NONE - """ + """Executes the job.""" + ... def eval(self) -> None: - """ - Evaluates the job. - - Parameters - ---------- - - Returns - ------- - NONE - """ + """Evaluates the job.""" + ... def finalize(self) -> None: - """ - Finalizes the job. - - Parameters - ---------- - - Returns - ------- - NONE - """ + """Finalizes the job.""" + ... def cancel(self) -> None: - """ - Cancels the job. - - Parameters - ---------- - - Returns - ------- - NONE - """ + """Cancels the job.""" + ... diff --git a/tests/unit/test_job_dao.py b/tests/unit/test_job_dao.py index a969427..d091171 100644 --- a/tests/unit/test_job_dao.py +++ b/tests/unit/test_job_dao.py @@ -26,15 +26,10 @@ @pytest.fixture def example_sqlite_job_dao() -> Generator: - """ - Creates sqlite engine. - - Parameters - ---------- + """Creates sqlite engine. - Returns - ------- - None + Yields: + Generator: sqlite engine """ engine = SQLJobDAO("sqlite+pysqlite://") yield engine @@ -42,60 +37,40 @@ def example_sqlite_job_dao() -> Generator: @pytest.fixture def example_job_status() -> JobStatusType: - """ - Creates a job status. + """Creates a job status. - Parameters - ---------- - - Returns - ------- - JobStatusType + Returns: + JobStatusType: job status for tests """ return JobStatusType.NOTSTARTET @pytest.fixture def example_exec_profile() -> ExecProfile: - """ - Creates an exec profile. + """Creates an exec profile. - Parameters - ---------- - - Returns - ------- - ExecProfile + Returns: + ExecProfile: exec profile for tests """ return ExecProfile(type_=ExecProfileType.PYTHON, wf_lang=WfLangType.CWL) @pytest.fixture def example_workflow() -> dict: - """ - Creates a workflow. + """Creates a workflow. - Parameters - ---------- - - Returns - ------- - dict + Returns: + dict: workflow """ return {"test": 1} @pytest.fixture def example_inputs() -> dict: - """ - Creates inputs. + """Creates inputs. - Parameters - ---------- - - Returns - ------- - dict + Returns: + dict: input parameters for the workflow """ return {"hello": "world"} @@ -108,25 +83,17 @@ def example_job_id( example_inputs: dict, example_sqlite_job_dao, ) -> UUID: - """ - Creates a job id. + """Creates a job id. - Parameters - ---------- - example_job_status: JobStatusType - status of the job - example_exec_profile: ExecProfile - execution profile of the job - example_workflow: dict - job's workflow - example_inputs: dict - input parameters of the workflow - example_sqlite_job_dao - database engine + Args: + example_job_status (JobStatusType): status of the job + example_exec_profile (ExecProfile): execution profile of the job + example_workflow (dict): job's workflow + example_inputs (dict): input parameters of the workflow + example_sqlite_job_dao (_type_): database engine - Returns - ------- - UUID + Returns: + UUID: job id """ return example_sqlite_job_dao.create( example_job_status, @@ -150,25 +117,14 @@ def test_create( example_inputs: dict, example_sqlite_job_dao, ) -> None: - """ - Tests the create method from SQLJobDAO class. + """Tests the create method from SQLJobDAO class. - Parameters - ---------- - example_job_status: JobStatusType - status of the job - example_exec_profile: ExecProfile - execution profile of the job - example_workflow: dict - job's workflow - example_inputs: dict - input parameters of the workflow - example_sqlite_job_dao - database engine - - Returns - ------- - None + Args: + example_job_status (JobStatusType): status of the job + example_exec_profile (ExecProfile): execution profile of the job + example_workflow (dict): job's workflow + example_inputs (dict): input parameters of the workflow + example_sqlite_job_dao (_type_): database engine """ job_id = example_sqlite_job_dao.create( example_job_status, @@ -198,7 +154,7 @@ def test_create( @pytest.mark.usefixtures( - "example_sqlite_job_dao", "example_job_id", "example_exec_profile" + "example_sqlite_job_dgao", "example_job_id", "example_exec_profile" ) def test_get( example_job_id: UUID, @@ -206,23 +162,13 @@ def test_get( example_exec_profile: ExecProfile, example_sqlite_job_dao, ) -> None: - """ - Tests the get method from SQLJobDAO class. - - Parameters - ---------- - example_job_id: UUID - id of the job - example_job_status: JobStatusType - status of the job - example_exec_profile: ExecProfile - execution profile of the job - example_sqlite_job_dao - database engine + """Tests the get method from SQLJobDAO class. - Returns - ------- - None + Args: + example_job_id (UUID): id of the job + example_job_status (JobStatusType): status of the job + example_exec_profile (ExecProfile): execution profile of the job + example_sqlite_job_dao (_type_): database engine """ job_id = example_job_id db_job = example_sqlite_job_dao.get(job_id) @@ -255,23 +201,13 @@ def test_update( example_inputs: dict, example_sqlite_job_dao, ) -> None: - """ - Tests the update method from SQLJobDAO class. - - Parameters - ---------- - example_job_id: UUID - id of the job - example_exec_profile: ExecProfile - execution profile of the job - example_inputs: dict - input parameters of the workflow - example_sqlite_job_dao: Engine - database engine + """Tests the update method from SQLJobDAO class. - Returns - ------- - None + Args: + example_job_id (UUID): id of the job + example_exec_profile (ExecProfile): execution profile of the job + example_inputs (dict): input parameters of the workflow + example_sqlite_job_dao (_type_): database engine """ job_id = example_job_id job = PythonJob(