From 007b591000fb686ffd5fc5d6957b2c00316e22fe Mon Sep 17 00:00:00 2001 From: alex-andrassy Date: Mon, 10 Jul 2023 09:33:41 +0100 Subject: [PATCH 01/10] Create roles.yml --- documentation/data/roles.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 documentation/data/roles.yml diff --git a/documentation/data/roles.yml b/documentation/data/roles.yml new file mode 100644 index 0000000..6d89cc5 --- /dev/null +++ b/documentation/data/roles.yml @@ -0,0 +1,14 @@ +- role: clinical lead + people: +- role: Clinical Safety Officer + people: +- role: csc_development_lead + people: +- role: csc_ml_lead + people: +- role: stakeholder + people: +- role: hod_for_deployment + people: +- role: staff_training_lead + people: \ No newline at end of file From 6939ae67baf9471a783af1c79fccf9eae1aabe73 Mon Sep 17 00:00:00 2001 From: alex-andrassy Date: Mon, 10 Jul 2023 10:45:41 +0100 Subject: [PATCH 02/10] Added data files just training_data outstanding --- documentation/data/device.yml | 18 ++++++++++++++ documentation/data/documents.yml | 40 ++++++++++++++++++++++++++++++++ documentation/data/model.yml | 10 ++++++++ 3 files changed, 68 insertions(+) create mode 100644 documentation/data/documents.yml create mode 100644 documentation/data/model.yml diff --git a/documentation/data/device.yml b/documentation/data/device.yml index 884287a..fd947b3 100644 --- a/documentation/data/device.yml +++ b/documentation/data/device.yml @@ -28,3 +28,21 @@ csc_website_url: #the GitHub URL github_url: +#path to logo +logo: + +# Name of or department that approved the project +Project_approver: + +# +Approval_date: + +#is project service development (bool) +Service_Dev: + +#path to king image for website +King_image: + +#expected completion date +exp_comp_date: + diff --git a/documentation/data/documents.yml b/documentation/data/documents.yml new file mode 100644 index 0000000..4092341 --- /dev/null +++ b/documentation/data/documents.yml @@ -0,0 +1,40 @@ +- document: proposal + version: +- document: software_req_spec + version: +- document: hazard_log + version: +- document: software_design_spec + version: +- document: design_plan + version: +- document: verif_and_validation_plan + version: +- document: user_acceptance_criteria + version: +- document: support_agreement + version: +- document: post_deployment_surveillance_plan + version: +- document: clinical_safety_case_report + version: +- document: clinical_risk_management_plan + version: +- document: cyber_security + version: +- document: revision_level_history + version: +- document: risk_management_report + version: +- document: software_description + version: +- document: software_development_and_maintenance_rec + version: +- document: software_plan + version: +- document: software_release_activity_record + version: +- document: test_record + version: +- document: unresolved_anomalies + version: \ No newline at end of file diff --git a/documentation/data/model.yml b/documentation/data/model.yml new file mode 100644 index 0000000..aa74bb1 --- /dev/null +++ b/documentation/data/model.yml @@ -0,0 +1,10 @@ +Model: + MLFlow artefacts: + - image: path/to/performance_image1.png + description: “text description” + - image: image2.png + description: “text description” + + model_version: “v2.0” + date: + pytorch_version: \ No newline at end of file From b90ab64ea5e76fef298f498b6471c7737aeb12f5 Mon Sep 17 00:00:00 2001 From: alex-andrassy Date: Mon, 10 Jul 2023 11:09:37 +0100 Subject: [PATCH 03/10] Update documents.yml --- documentation/data/documents.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/documentation/data/documents.yml b/documentation/data/documents.yml index 4092341..f161216 100644 --- a/documentation/data/documents.yml +++ b/documentation/data/documents.yml @@ -37,4 +37,6 @@ - document: test_record version: - document: unresolved_anomalies + version: +- document: data_description version: \ No newline at end of file From 2a2cbe068d10fb0a206fb2b11b4e6a8078d2e607 Mon Sep 17 00:00:00 2001 From: alex-andrassy Date: Tue, 11 Jul 2023 10:46:37 +0100 Subject: [PATCH 04/10] Update device.yml --- documentation/data/device.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/data/device.yml b/documentation/data/device.yml index fd947b3..91601c3 100644 --- a/documentation/data/device.yml +++ b/documentation/data/device.yml @@ -46,3 +46,4 @@ King_image: #expected completion date exp_comp_date: +statement_of_purpose: From f8da48f77bd69914e81f6e15e0b612e900af609d Mon Sep 17 00:00:00 2001 From: alex-andrassy Date: Tue, 11 Jul 2023 11:07:13 +0100 Subject: [PATCH 05/10] add copy file github action --- .github/workflows/copy_file.yml | 26 ++++++++++++++++++++++++++ documentation/data/website.yml | 11 +++++++++++ 2 files changed, 37 insertions(+) create mode 100644 .github/workflows/copy_file.yml create mode 100644 documentation/data/website.yml diff --git a/.github/workflows/copy_file.yml b/.github/workflows/copy_file.yml new file mode 100644 index 0000000..63b7553 --- /dev/null +++ b/.github/workflows/copy_file.yml @@ -0,0 +1,26 @@ +name: copy_file + +on: + push: + branches: + - master + +jobs: + copy-file: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Pushes file + uses: dmnemec/copy_file_to_another_repo_action@main + env: + API_TOKEN_GITHUB: ${{ secrets.API_TOKEN_GITHUB }} + with: + source_file: './documentation/data/website.yml' + destination_repo: 'GSTT-CSC/gstt-csc.github.io' + destination_folder: '_projects' + #email and username of the user associated with the token + user_email: '' + user_name: '' + commit_message: '' \ No newline at end of file diff --git a/documentation/data/website.yml b/documentation/data/website.yml new file mode 100644 index 0000000..a4c4cc3 --- /dev/null +++ b/documentation/data/website.yml @@ -0,0 +1,11 @@ +statement_of_purpose: 'Purpose' + +device_name: DEVICE + +logo: path/to/logo + +MLFlow_artefacts: + - image: path/to/performance_image1.png + description: “text description” + +Dataset_size: X From ed4def73c907fee59ca21e8b0f1c75304e9695b2 Mon Sep 17 00:00:00 2001 From: alex-andrassy Date: Fri, 14 Jul 2023 11:06:57 +0100 Subject: [PATCH 06/10] Update training_data.yml --- documentation/data/training_data.yml | 30 ++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/documentation/data/training_data.yml b/documentation/data/training_data.yml index 0de8e01..0c515d1 100644 --- a/documentation/data/training_data.yml +++ b/documentation/data/training_data.yml @@ -5,6 +5,36 @@ dataset_version: preprocess_description: train_dataset: +- group: total + size: + - grouping: gender + Female: + Male: + - grouping: ethnicity + not_stated: + asian: + black: + latin_american: + mixed: + white: + - grouping: age + 0-15: + 16-20: + 21-25: + 26-30: + 31-35: + 36-40: + 41-45: + 46-50: + 51-55: + 56-60: + 61-65: + 66-70: + 71-75: + 76-80: + 81-85: + 86-90: + 91_plus: test_dataset: From cb06a69138935b9eb97f1fee82d6272503f1da07 Mon Sep 17 00:00:00 2001 From: alex-andrassy Date: Fri, 14 Jul 2023 15:09:12 +0100 Subject: [PATCH 07/10] add website mds --- documentation/documents/website_modelcard.md | 27 +++++++++++++++++++ documentation/documents/website_project.md | 28 ++++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 documentation/documents/website_modelcard.md create mode 100644 documentation/documents/website_project.md diff --git a/documentation/documents/website_modelcard.md b/documentation/documents/website_modelcard.md new file mode 100644 index 0000000..5b23b03 --- /dev/null +++ b/documentation/documents/website_modelcard.md @@ -0,0 +1,27 @@ +--- +layout: project_page +title: {{device.name}} +image: /assets/img/projects/xray_carpel.jpg +summary: A computer aided diagnosis tool which would automatically run when a scaphoid fracture is suspected. +csc-lead: Dika +modality: X-ray +pathology: Occult carpal fractures (wrist and hands) +rationale: MRI is superior in identification of occult carpal fracture, but is not always accessible. An AI tool to aide clinical diagnosis of occult carpal fractures using x-rays would increase diagnostic sensitivity in areas and situations where MRI is not available. +patient-pathway: A&E attendance with a suspected scaphoid fracture or hand/wrist injury. +training-data: Patient cohort identified using CogStack looking for phrases 'scaphoid' and 'MRI clinical'. The query generated a list of approximately 1000 patients who have had an MRI, with the view that all patients who had an MRI will also have had an x-ray. The patient list is now being stratified to `test`, `control` and `back-up` groups. +errors: Needless referrals to fracture clinic and needless immobilisation in patients who do not have a fracture; repeated x-rays; referrals to MRI. +goals: An automated computer aided diagnosis tool triggered when appropriate x-rays are taken. +success-criteria: Improvement in diagnostic accuracy and diagnostic speed. +alternatives: Gleamer, which specialise in trauma x-rays, has been considered for this purpose but was decided not suitable to solve this particular clinical problem. The decision was made to train an in-house algorithm instead +--- +MRI imaging is superior in identification of occult carpal fracture, but is not always accessible. Imaging from X-rays can give suboptimal views, and the presentation of arthritis can make small fractures difficult to see. An AI tool to aide clinical diagnosis of occult carpal fractures using x-rays would increase diagnostic sensitivity in areas and situations where MRI is not available. +A computer aided diagnosis tool which would automatically run when either a scaphoid fracture is suspected or if a patient is referred for a hand/wrist x-ray from A&E would increase sensitivity and confidence of diagnosis. Carpal fractures can be difficult to identify and patients with high clinical suspicion are put in a splint and referred to the fracture clinic even if a fracture isn’t seen on the x-ray by the clinician. Subtle lucency of an un-displaced fracture and the significance of a small bone fragment is currently easily missed. A successful tool would therefore increase diagnostic confidence and accuracy and reduce repeated x-rays and needless fracture clinic referrals +
+
+Clinical lead: Davina Mak
+
+**Project Plan**
+1. Meeting of all persons involved to determine AI specifications.

2. Setting technical and system requirements for AI model.
3. Dataset curation (retrospective).

4. Model training

5. Model testing

6. Implementation

7. Audit +
+
+References:
TOHETI trial results \ No newline at end of file diff --git a/documentation/documents/website_project.md b/documentation/documents/website_project.md new file mode 100644 index 0000000..80bd391 --- /dev/null +++ b/documentation/documents/website_project.md @@ -0,0 +1,28 @@ +--- +layout: project_page +title: X-Ray Occult Carpal Detection +status: Developing +image: /assets/img/projects/xray_carpel.jpg +summary: A computer aided diagnosis tool which would automatically run when a scaphoid fracture is suspected. +csc-lead: Dika +modality: X-ray +pathology: Occult carpal fractures (wrist and hands) +rationale: MRI is superior in identification of occult carpal fracture, but is not always accessible. An AI tool to aide clinical diagnosis of occult carpal fractures using x-rays would increase diagnostic sensitivity in areas and situations where MRI is not available. +patient-pathway: A&E attendance with a suspected scaphoid fracture or hand/wrist injury. +training-data: Patient cohort identified using CogStack looking for phrases 'scaphoid' and 'MRI clinical'. The query generated a list of approximately 1000 patients who have had an MRI, with the view that all patients who had an MRI will also have had an x-ray. The patient list is now being stratified to `test`, `control` and `back-up` groups. +errors: Needless referrals to fracture clinic and needless immobilisation in patients who do not have a fracture; repeated x-rays; referrals to MRI. +goals: An automated computer aided diagnosis tool triggered when appropriate x-rays are taken. +success-criteria: Improvement in diagnostic accuracy and diagnostic speed. +alternatives: Gleamer, which specialise in trauma x-rays, has been considered for this purpose but was decided not suitable to solve this particular clinical problem. The decision was made to train an in-house algorithm instead +--- +MRI imaging is superior in identification of occult carpal fracture, but is not always accessible. Imaging from X-rays can give suboptimal views, and the presentation of arthritis can make small fractures difficult to see. An AI tool to aide clinical diagnosis of occult carpal fractures using x-rays would increase diagnostic sensitivity in areas and situations where MRI is not available. +A computer aided diagnosis tool which would automatically run when either a scaphoid fracture is suspected or if a patient is referred for a hand/wrist x-ray from A&E would increase sensitivity and confidence of diagnosis. Carpal fractures can be difficult to identify and patients with high clinical suspicion are put in a splint and referred to the fracture clinic even if a fracture isn’t seen on the x-ray by the clinician. Subtle lucency of an un-displaced fracture and the significance of a small bone fragment is currently easily missed. A successful tool would therefore increase diagnostic confidence and accuracy and reduce repeated x-rays and needless fracture clinic referrals +
+
+Clinical lead: Davina Mak
+
+**Project Plan**
+1. Meeting of all persons involved to determine AI specifications.

2. Setting technical and system requirements for AI model.
3. Dataset curation (retrospective).

4. Model training

5. Model testing

6. Implementation

7. Audit +
+
+References:
TOHETI trial results \ No newline at end of file From 060997efac8612315ddb61ffb5b777b7767342ba Mon Sep 17 00:00:00 2001 From: alex-andrassy Date: Fri, 14 Jul 2023 16:54:01 +0100 Subject: [PATCH 08/10] website mds now reference data files --- documentation/data/device.yml | 17 ++++++--- documentation/data/model.yml | 2 ++ documentation/data/training_data.yml | 1 + documentation/data/website.yml | 17 +++++---- documentation/documents/website_modelcard.md | 37 ++++++++++---------- documentation/documents/website_project.md | 28 ++++++++------- 6 files changed, 58 insertions(+), 44 deletions(-) diff --git a/documentation/data/device.yml b/documentation/data/device.yml index 91601c3..536f8da 100644 --- a/documentation/data/device.yml +++ b/documentation/data/device.yml @@ -6,6 +6,18 @@ name: "DEVICE" # Shorthand Name for device, (max 5 characters) name_sh: "DEV" +statement_of_purpose: PURPOSE + +summary: SUMMARY + +modality: X-ray + +pathology: PATHOLOGY + +rationale: RATIONALE + +patient_pathway: PATHWAY + # The current release number for the device version: "v0.1.0" @@ -40,10 +52,7 @@ Approval_date: #is project service development (bool) Service_Dev: -#path to king image for website -King_image: - #expected completion date exp_comp_date: -statement_of_purpose: + diff --git a/documentation/data/model.yml b/documentation/data/model.yml index aa74bb1..50ffd33 100644 --- a/documentation/data/model.yml +++ b/documentation/data/model.yml @@ -1,5 +1,7 @@ Model: MLFlow artefacts: + # will this path be in project or website repo or elsewhere? + # not sure how to handle handle these for website... - image: path/to/performance_image1.png description: “text description” - image: image2.png diff --git a/documentation/data/training_data.yml b/documentation/data/training_data.yml index 0c515d1..310af4b 100644 --- a/documentation/data/training_data.yml +++ b/documentation/data/training_data.yml @@ -5,6 +5,7 @@ dataset_version: preprocess_description: train_dataset: + summary: DATASET SUMMARY - group: total size: - grouping: gender diff --git a/documentation/data/website.yml b/documentation/data/website.yml index a4c4cc3..47123fc 100644 --- a/documentation/data/website.yml +++ b/documentation/data/website.yml @@ -1,11 +1,10 @@ -statement_of_purpose: 'Purpose' - -device_name: DEVICE - +project_title: PROJECT TITLE +status: Developing +# image path in website repo +main_image_path: /assets/img/projects/IMAGE_TITLE.jpg +# logo path in website repo logo: path/to/logo -MLFlow_artefacts: - - image: path/to/performance_image1.png - description: “text description” - -Dataset_size: X +errors: ERRORS +goals: GOALS +success_criteria: CRITERIA diff --git a/documentation/documents/website_modelcard.md b/documentation/documents/website_modelcard.md index 5b23b03..b083ba1 100644 --- a/documentation/documents/website_modelcard.md +++ b/documentation/documents/website_modelcard.md @@ -1,27 +1,28 @@ --- layout: project_page -title: {{device.name}} -image: /assets/img/projects/xray_carpel.jpg -summary: A computer aided diagnosis tool which would automatically run when a scaphoid fracture is suspected. +title: {{website.project_title}} +name: {{device.name}} +status: {{website.status}} +image: {{website.main_image_path}} +summary: {{device.summary}} +modality: {{device.modality}} +pathology: {{device.pathology}} +rationale: {{device.rationale}} +patient-pathway: {{device.patient_pathway}} +training-data: {{training_data.train_dataset.summary}} +errors: {{website.errors}} +goals: {{website.goals}} +success-criteria: {{website.success_criteria}} + csc-lead: Dika -modality: X-ray -pathology: Occult carpal fractures (wrist and hands) -rationale: MRI is superior in identification of occult carpal fracture, but is not always accessible. An AI tool to aide clinical diagnosis of occult carpal fractures using x-rays would increase diagnostic sensitivity in areas and situations where MRI is not available. -patient-pathway: A&E attendance with a suspected scaphoid fracture or hand/wrist injury. -training-data: Patient cohort identified using CogStack looking for phrases 'scaphoid' and 'MRI clinical'. The query generated a list of approximately 1000 patients who have had an MRI, with the view that all patients who had an MRI will also have had an x-ray. The patient list is now being stratified to `test`, `control` and `back-up` groups. -errors: Needless referrals to fracture clinic and needless immobilisation in patients who do not have a fracture; repeated x-rays; referrals to MRI. -goals: An automated computer aided diagnosis tool triggered when appropriate x-rays are taken. -success-criteria: Improvement in diagnostic accuracy and diagnostic speed. -alternatives: Gleamer, which specialise in trauma x-rays, has been considered for this purpose but was decided not suitable to solve this particular clinical problem. The decision was made to train an in-house algorithm instead +alternatives: Gleamer, which specialise in trauma x-rays, has been considered for this purpose but was decided not suitable to solve this particular clinical problem. The decision was made to train an in-house algorithm instead. + + + + --- MRI imaging is superior in identification of occult carpal fracture, but is not always accessible. Imaging from X-rays can give suboptimal views, and the presentation of arthritis can make small fractures difficult to see. An AI tool to aide clinical diagnosis of occult carpal fractures using x-rays would increase diagnostic sensitivity in areas and situations where MRI is not available. A computer aided diagnosis tool which would automatically run when either a scaphoid fracture is suspected or if a patient is referred for a hand/wrist x-ray from A&E would increase sensitivity and confidence of diagnosis. Carpal fractures can be difficult to identify and patients with high clinical suspicion are put in a splint and referred to the fracture clinic even if a fracture isn’t seen on the x-ray by the clinician. Subtle lucency of an un-displaced fracture and the significance of a small bone fragment is currently easily missed. A successful tool would therefore increase diagnostic confidence and accuracy and reduce repeated x-rays and needless fracture clinic referrals

-Clinical lead: Davina Mak
-
-**Project Plan**
-1. Meeting of all persons involved to determine AI specifications.

2. Setting technical and system requirements for AI model.
3. Dataset curation (retrospective).

4. Model training

5. Model testing

6. Implementation

7. Audit -
-
References:
TOHETI trial results \ No newline at end of file diff --git a/documentation/documents/website_project.md b/documentation/documents/website_project.md index 80bd391..1b27270 100644 --- a/documentation/documents/website_project.md +++ b/documentation/documents/website_project.md @@ -1,25 +1,27 @@ --- layout: project_page -title: X-Ray Occult Carpal Detection -status: Developing -image: /assets/img/projects/xray_carpel.jpg -summary: A computer aided diagnosis tool which would automatically run when a scaphoid fracture is suspected. +title: {{website.project_title}} +status: {{website.status}} +image: {{website.main_image_path}} +summary: {{device.summary}} +modality: {{device.modality}} +pathology: {{device.pathology}} +rationale: {{device.rationale}} +patient-pathway: {{device.patient_pathway}} +training-data: {{training_data.train_dataset.summary}} +errors: {{website.errors}} +goals: {{website.goals}} +success-criteria: {{website.success_criteria}} + csc-lead: Dika -modality: X-ray -pathology: Occult carpal fractures (wrist and hands) -rationale: MRI is superior in identification of occult carpal fracture, but is not always accessible. An AI tool to aide clinical diagnosis of occult carpal fractures using x-rays would increase diagnostic sensitivity in areas and situations where MRI is not available. -patient-pathway: A&E attendance with a suspected scaphoid fracture or hand/wrist injury. -training-data: Patient cohort identified using CogStack looking for phrases 'scaphoid' and 'MRI clinical'. The query generated a list of approximately 1000 patients who have had an MRI, with the view that all patients who had an MRI will also have had an x-ray. The patient list is now being stratified to `test`, `control` and `back-up` groups. -errors: Needless referrals to fracture clinic and needless immobilisation in patients who do not have a fracture; repeated x-rays; referrals to MRI. -goals: An automated computer aided diagnosis tool triggered when appropriate x-rays are taken. -success-criteria: Improvement in diagnostic accuracy and diagnostic speed. alternatives: Gleamer, which specialise in trauma x-rays, has been considered for this purpose but was decided not suitable to solve this particular clinical problem. The decision was made to train an in-house algorithm instead + --- MRI imaging is superior in identification of occult carpal fracture, but is not always accessible. Imaging from X-rays can give suboptimal views, and the presentation of arthritis can make small fractures difficult to see. An AI tool to aide clinical diagnosis of occult carpal fractures using x-rays would increase diagnostic sensitivity in areas and situations where MRI is not available. A computer aided diagnosis tool which would automatically run when either a scaphoid fracture is suspected or if a patient is referred for a hand/wrist x-ray from A&E would increase sensitivity and confidence of diagnosis. Carpal fractures can be difficult to identify and patients with high clinical suspicion are put in a splint and referred to the fracture clinic even if a fracture isn’t seen on the x-ray by the clinician. Subtle lucency of an un-displaced fracture and the significance of a small bone fragment is currently easily missed. A successful tool would therefore increase diagnostic confidence and accuracy and reduce repeated x-rays and needless fracture clinic referrals

-Clinical lead: Davina Mak
+Clinical lead: {% for r in roles %}{% if r.role == "clinical lead" %}{{r.people | join('
, ')}}{% endif %}{% endfor %}

**Project Plan**
1. Meeting of all persons involved to determine AI specifications.

2. Setting technical and system requirements for AI model.
3. Dataset curation (retrospective).

4. Model training

5. Model testing

6. Implementation

7. Audit From 69093d326a8ee55257825f120c1e6e7f9cbf52e8 Mon Sep 17 00:00:00 2001 From: alex-andrassy Date: Wed, 19 Jul 2023 09:24:02 +0100 Subject: [PATCH 09/10] add render and push action for modelcard --- .github/workflows/copy_file.yml | 8 +-- .github/workflows/render_and_push.yml | 42 +++++++++++++++ documentation/data/training_data.yml | 75 +++++++++++++++------------ documentation/plotyml.py | 35 +++++++++++++ 4 files changed, 122 insertions(+), 38 deletions(-) create mode 100644 .github/workflows/render_and_push.yml create mode 100644 documentation/plotyml.py diff --git a/.github/workflows/copy_file.yml b/.github/workflows/copy_file.yml index 63b7553..6f982e3 100644 --- a/.github/workflows/copy_file.yml +++ b/.github/workflows/copy_file.yml @@ -1,9 +1,9 @@ name: copy_file -on: - push: - branches: - - master +on: workflow_dispatch +# push: +# branches: +# - master jobs: copy-file: diff --git a/.github/workflows/render_and_push.yml b/.github/workflows/render_and_push.yml new file mode 100644 index 0000000..eceb0ab --- /dev/null +++ b/.github/workflows/render_and_push.yml @@ -0,0 +1,42 @@ +# this is the version that works +name: render_and_push + +on: workflow_dispatch +# push: +# branches: +# - main + +jobs: + update-and-copy-md: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Environment + run: | + python -m venv testenv + . testenv/bin/activate + pip install --upgrade pip + pip install rdm + make ./release/website_modelcard.md + mkdir md-to-send + mv ./release/website_modelcard.md ./md-to-send/website_modelcard.md + - name: Pushes test file + uses: SpinyOwl/publish-artifact-to-git@1.0.1 + with: + # create a personal access token with read/write access to both repos, and save it to secrets. access tokens expire after 60 days. + github_pat: '${{ secrets.MODEL_CARD_DEMO }}' +# github_token: '${{ secrets.GITHUB_TOKEN }}' + repository: alex-andrassy/modelcard_demo + branch: main + source_folder: ./md-to-send +# target_folder: data + no_delete: true + +# works but get this Warning: The `set-output` command is deprecated and will be disabled soon... +# Please upgrade to using Environment Files... +# For more information see: https://github.blog/changelog/2022-10-11-github-actions-deprecating-save-state-and-set-output-commands/ +# link says command disable is postponed because of high use +# IF THIS COMMAND IS DISABLED AND THE ACTION IS NOT UPDATED: +# A fix could be to produce the .md file and images(?) the same way and upload as an artifact, then there are many actions which can be used to... +# download the artifact from the project repo to the website repo. OR, send the data over, and produce the .md in the website repo. \ No newline at end of file diff --git a/documentation/data/training_data.yml b/documentation/data/training_data.yml index 310af4b..3baf586 100644 --- a/documentation/data/training_data.yml +++ b/documentation/data/training_data.yml @@ -1,41 +1,48 @@ -xnat_project_name: +xnat_project_name: NAME -dataset_version: +dataset_version: VERSION -preprocess_description: +preprocess_description: DESCRIPTION train_dataset: - summary: DATASET SUMMARY -- group: total - size: - - grouping: gender - Female: - Male: - - grouping: ethnicity - not_stated: - asian: - black: - latin_american: - mixed: - white: - - grouping: age - 0-15: - 16-20: - 21-25: - 26-30: - 31-35: - 36-40: - 41-45: - 46-50: - 51-55: - 56-60: - 61-65: - 66-70: - 71-75: - 76-80: - 81-85: - 86-90: - 91_plus: +- total: + summary: DATASET SUMMARY + size: SIZE + subgroups: + - gender: + Female: 3 + Male: 1 + - ethnicity: + not_stated: + asian: + black: + latin_american: + mixed: + white: + - age: + 0-15: + 16-20: + 21-25: + 26-30: + 31-35: + 36-40: + 41-45: + 46-50: + 51-55: + 56-60: + 61-65: + 66-70: + 71-75: + 76-80: + 81-85: + 86-90: + 91_plus: +- positive: + subgroups: +- postive_female_mixed: + subgroups: + - age: + test_dataset: diff --git a/documentation/plotyml.py b/documentation/plotyml.py new file mode 100644 index 0000000..2320354 --- /dev/null +++ b/documentation/plotyml.py @@ -0,0 +1,35 @@ +import matplotlib.pyplot as plt +import yaml + +with open('./data/training_data.yml') as file: + data = yaml.safe_load(file) + +print(data) + +w = data['train_dataset'][0]['total']['subgroups'][0]['gender'] +y = w.values() +x = w.keys() +#[0]['subgroups'][0].values() +print(y) +print(x) + +# dict_keys is iterable but not indexable, so below gives way to index +# ATA - see favourites for recommended way +#for key in y: +# print(key) +# z= data['train_dataset'][0][key] +# print(z) + +fig, ax = plt.subplots() + +#bar_labels = ['red', 'blue'] +#bar_colors = ['tab:red', 'tab:blue'] + +ax.bar(x, y) + +ax.set_ylabel('Patients') +ax.set_title('Patient demographics') +ax.set_yticklabels(tick_labels.astype(int)) +#ax.legend(title='Fruit color') + +plt.show() \ No newline at end of file From 6afbbb7f6442f3a10dbbb932bfba57b68cae1812 Mon Sep 17 00:00:00 2001 From: alex-andrassy Date: Fri, 28 Jul 2023 14:49:57 +0100 Subject: [PATCH 10/10] update render and push addresses issues 51 and 54. render and push action should push modelcard .md file to _projects dir in website repo, and demographics graphs to assets/img/projects dir in website repo. Currently set up to push to 228_add_modelcard_layout branch of website repo for testing --- .github/workflows/render_and_push.yml | 45 +++-- documentation/data/device.yml | 4 +- documentation/data/model.yml | 22 +-- documentation/data/training_data.yml | 27 +-- documentation/data/website.yml | 3 + documentation/documents/website_modelcard.md | 21 ++- documentation/plotyml.py | 180 ++++++++++++++++--- 7 files changed, 235 insertions(+), 67 deletions(-) diff --git a/.github/workflows/render_and_push.yml b/.github/workflows/render_and_push.yml index eceb0ab..4754775 100644 --- a/.github/workflows/render_and_push.yml +++ b/.github/workflows/render_and_push.yml @@ -1,6 +1,7 @@ -# this is the version that works +# make sure main_image_path in website.yml is correct name: render_and_push +# left as manual for testing, to be changed to on push on: workflow_dispatch # push: # branches: @@ -12,31 +13,49 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 - - name: Environment + - name: render md and graphs + # set up environment, render website_modelcard.md with yml data in data directory, and plot demographics data + # NOTE to fix - currently fails if demo_graphs dir already exists in repo AND has files in run: | python -m venv testenv . testenv/bin/activate pip install --upgrade pip pip install rdm + pip install matplotlib + pip install pillow + cd ./documentation make ./release/website_modelcard.md - mkdir md-to-send - mv ./release/website_modelcard.md ./md-to-send/website_modelcard.md - - name: Pushes test file + mkdir ../md-to-send + mv ./release/website_modelcard.md ../md-to-send/${GITHUB_REPOSITORY#*/}_modelcard.md + mkdir demo_graphs + python plotyml.py + - name: Pushes md uses: SpinyOwl/publish-artifact-to-git@1.0.1 with: - # create a personal access token with read/write access to both repos, and save it to secrets. access tokens expire after 60 days. + # need to create a personal access token with read/write access to website repo, and save it to secrets. access tokens expire after 60 days. + # a user based naming convention for secrets may be necessary, to ensure each user calls their own access token (if multiple access tokens saved to a repo's secrets) github_pat: '${{ secrets.MODEL_CARD_DEMO }}' -# github_token: '${{ secrets.GITHUB_TOKEN }}' - repository: alex-andrassy/modelcard_demo - branch: main + repository: GSTT-CSC/gstt-csc.github.io + # sending to branch for testing, to be changed to main/master + branch: 228_add_modelcard_layout + # source folder path needs to match the md-to-send directory created in render md and graphs step above source_folder: ./md-to-send -# target_folder: data + # set target folder in website repo for modelcard md file + target_folder: _projects + no_delete: true + - name: Pushes graphs + uses: SpinyOwl/publish-artifact-to-git@1.0.1 + with: + github_pat: '${{ secrets.MODEL_CARD_DEMO }}' + repository: GSTT-CSC/gstt-csc.github.io + # sending to branch for testing, to be changed to main/master + branch: 228_add_modelcard_layout + # source folder path needs to match the demo_graphs directory created in render md and graphs step above + source_folder: ./documentation/demo_graphs + target_folder: assets/img/projects no_delete: true # works but get this Warning: The `set-output` command is deprecated and will be disabled soon... # Please upgrade to using Environment Files... # For more information see: https://github.blog/changelog/2022-10-11-github-actions-deprecating-save-state-and-set-output-commands/ # link says command disable is postponed because of high use -# IF THIS COMMAND IS DISABLED AND THE ACTION IS NOT UPDATED: -# A fix could be to produce the .md file and images(?) the same way and upload as an artifact, then there are many actions which can be used to... -# download the artifact from the project repo to the website repo. OR, send the data over, and produce the .md in the website repo. \ No newline at end of file diff --git a/documentation/data/device.yml b/documentation/data/device.yml index 536f8da..f72e052 100644 --- a/documentation/data/device.yml +++ b/documentation/data/device.yml @@ -10,7 +10,7 @@ statement_of_purpose: PURPOSE summary: SUMMARY -modality: X-ray +modality: MODALITY pathology: PATHOLOGY @@ -34,7 +34,7 @@ safety_class: B # "III" - mhra_class: "IIb" -# link ot the website page +# link to the website page csc_website_url: #the GitHub URL diff --git a/documentation/data/model.yml b/documentation/data/model.yml index 50ffd33..53d060a 100644 --- a/documentation/data/model.yml +++ b/documentation/data/model.yml @@ -1,12 +1,12 @@ -Model: - MLFlow artefacts: - # will this path be in project or website repo or elsewhere? - # not sure how to handle handle these for website... - - image: path/to/performance_image1.png - description: “text description” - - image: image2.png - description: “text description” +model_architecture: "model" - model_version: “v2.0” - date: - pytorch_version: \ No newline at end of file +MLFlow artefacts: +# will this path be in project or website repo or elsewhere? +- image: path/to/performance_image1.png + description: "text description" +- image: image2.png + description: "text description" + +model_version: "v2.0" +date: +pytorch_version: \ No newline at end of file diff --git a/documentation/data/training_data.yml b/documentation/data/training_data.yml index 3baf586..4bb08f6 100644 --- a/documentation/data/training_data.yml +++ b/documentation/data/training_data.yml @@ -4,22 +4,25 @@ dataset_version: VERSION preprocess_description: DESCRIPTION +train_dataset_summary: DATASET SUMMARY +train_data_excl: Exclusion Criteria +train_dataset_size: SIZE +train_input: Images + train_dataset: - total: - summary: DATASET SUMMARY - size: SIZE subgroups: - gender: - Female: 3 - Male: 1 + Female: 10 + Male: 10 - ethnicity: - not_stated: - asian: - black: - latin_american: - mixed: - white: - - age: + Not Stated: + Asian: + Black: + Latin American: + Mixed: + White: + - Age: 0-15: 16-20: 21-25: @@ -36,7 +39,7 @@ train_dataset: 76-80: 81-85: 86-90: - 91_plus: + 91+: - positive: subgroups: - postive_female_mixed: diff --git a/documentation/data/website.yml b/documentation/data/website.yml index 47123fc..281a2af 100644 --- a/documentation/data/website.yml +++ b/documentation/data/website.yml @@ -1,10 +1,13 @@ project_title: PROJECT TITLE status: Developing + # image path in website repo main_image_path: /assets/img/projects/IMAGE_TITLE.jpg + # logo path in website repo logo: path/to/logo errors: ERRORS goals: GOALS success_criteria: CRITERIA + diff --git a/documentation/documents/website_modelcard.md b/documentation/documents/website_modelcard.md index b083ba1..547cfde 100644 --- a/documentation/documents/website_modelcard.md +++ b/documentation/documents/website_modelcard.md @@ -1,28 +1,35 @@ --- -layout: project_page +layout: modelcard title: {{website.project_title}} -name: {{device.name}} +devicename: {{device.name}} status: {{website.status}} image: {{website.main_image_path}} +sex_graph: /assets/img/projects/{{device.name}}_sex_graph.png +ethn_graph: /assets/img/projects/{{device.name}}_ethn_graph.png +age_graph: /assets/img/projects/{{device.name}}_age_graph.png summary: {{device.summary}} modality: {{device.modality}} +model: {{model.model_architecture}} pathology: {{device.pathology}} rationale: {{device.rationale}} patient-pathway: {{device.patient_pathway}} -training-data: {{training_data.train_dataset.summary}} +training-data: {{training_data.train_dataset_summary}} +train_data_excl: {{training_data.train_data_excl}} +train_input: {{training_data.train_input}} +train_size: {{training_data.train_dataset_size}} errors: {{website.errors}} goals: {{website.goals}} success-criteria: {{website.success_criteria}} -csc-lead: Dika alternatives: Gleamer, which specialise in trauma x-rays, has been considered for this purpose but was decided not suitable to solve this particular clinical problem. The decision was made to train an in-house algorithm instead. --- -MRI imaging is superior in identification of occult carpal fracture, but is not always accessible. Imaging from X-rays can give suboptimal views, and the presentation of arthritis can make small fractures difficult to see. An AI tool to aide clinical diagnosis of occult carpal fractures using x-rays would increase diagnostic sensitivity in areas and situations where MRI is not available. -A computer aided diagnosis tool which would automatically run when either a scaphoid fracture is suspected or if a patient is referred for a hand/wrist x-ray from A&E would increase sensitivity and confidence of diagnosis. Carpal fractures can be difficult to identify and patients with high clinical suspicion are put in a splint and referred to the fracture clinic even if a fracture isn’t seen on the x-ray by the clinician. Subtle lucency of an un-displaced fracture and the significance of a small bone fragment is currently easily missed. A successful tool would therefore increase diagnostic confidence and accuracy and reduce repeated x-rays and needless fracture clinic referrals + +**Project Plan** +
1. Meeting of all persons involved to determine AI specifications.

2. Setting technical and system requirements for AI model.
3. Dataset curation (retrospective).

4. Model training

5. Model testing

6. Implementation

7. Audit

-References:
TOHETI trial results \ No newline at end of file +References:
\ No newline at end of file diff --git a/documentation/plotyml.py b/documentation/plotyml.py index 2320354..9f25673 100644 --- a/documentation/plotyml.py +++ b/documentation/plotyml.py @@ -1,35 +1,171 @@ import matplotlib.pyplot as plt import yaml +from PIL import Image, ImageDraw +# read data from yaml file with open('./data/training_data.yml') as file: data = yaml.safe_load(file) +# needed so that the graphs can be given device specific names in website repo +with open('./data/device.yml') as file2: + device = yaml.safe_load(file2) +name = device['name'] -print(data) +# plot sex graph +# would be better to have all plotting in for loop, when doing this note dict_keys is iterable but not indexable +try: + w = data['train_dataset'][0]['total']['subgroups'][0]['gender'] + y = w.values() + x = w.keys() -w = data['train_dataset'][0]['total']['subgroups'][0]['gender'] -y = w.values() -x = w.keys() -#[0]['subgroups'][0].values() -print(y) -print(x) + fig, ax = plt.subplots() + fig.set_size_inches(5.6, 4.8) + bars = ax.bar(x, y, color=(0.06, 0.38, 0.94, 0.7)) -# dict_keys is iterable but not indexable, so below gives way to index -# ATA - see favourites for recommended way -#for key in y: -# print(key) -# z= data['train_dataset'][0][key] -# print(z) +# formatting graph + ax.spines['top'].set_visible(False) + ax.spines['right'].set_visible(False) + ax.spines['left'].set_visible(False) + ax.spines['bottom'].set_color('#DDDDDD') -fig, ax = plt.subplots() + ax.tick_params(bottom=False, left=False) -#bar_labels = ['red', 'blue'] -#bar_colors = ['tab:red', 'tab:blue'] + ax.set_axisbelow(True) + ax.yaxis.grid(False) + ax.xaxis.grid(False) -ax.bar(x, y) +# add text above bars, same color as bars + bar_color = bars[0].get_facecolor() -ax.set_ylabel('Patients') -ax.set_title('Patient demographics') -ax.set_yticklabels(tick_labels.astype(int)) -#ax.legend(title='Fruit color') + for bar in bars: + ax.text( + bar.get_x() + bar.get_width() / 2, + # may want to add a small number to this param to give a small gap between text and bar + bar.get_height(), + bar.get_height(), + horizontalalignment='center', + color=bar_color, + weight='bold' + ) + + ax.set_xlabel('Sex', labelpad=15, color='#333333') + ax.set_ylabel('Number of Patients', labelpad=15, color='#333333') + ax.set_title('Patients in Training Dataset by Sex', pad=15, color='#333333', + weight='bold') + fig.tight_layout() + +# save fig and clear for next one + plt.savefig("demo_graphs/"+name+"_sex_graph.png") + plt.clf() + +except: + img = Image.new(mode="RGBA", size=(480,480), color='lightgrey') + draw = ImageDraw.Draw(img) + text = "No Image Available" + draw.text((10,10), text, fill=(0,0,0)) + img.save("demo_graphs/"+name+"_sex_graph.png") + + +# plot ethnicity graph +try: + w = data['train_dataset'][0]['total']['subgroups'][1]['ethnicity'] + y = w.values() + x = w.keys() + + fig, ax = plt.subplots() + fig.set_size_inches(8, 4.8) + bars = ax.bar(x, y, color=(0.06, 0.38, 0.94, 0.7)) + + # formatting graph + ax.spines['top'].set_visible(False) + ax.spines['right'].set_visible(False) + ax.spines['left'].set_visible(False) + ax.spines['bottom'].set_color('#DDDDDD') + + ax.tick_params(bottom=False, left=False) + + ax.set_axisbelow(True) + ax.yaxis.grid(False) + ax.xaxis.grid(False) + + # add text above bars, same color as bars + bar_color = bars[0].get_facecolor() + + for bar in bars: + ax.text( + bar.get_x() + bar.get_width() / 2, + bar.get_height(), + bar.get_height(), + horizontalalignment='center', + color=bar_color, + weight='bold' + ) + + ax.set_xlabel('Ethnicity', labelpad=15, color='#333333') + ax.set_ylabel('Number of Patients', labelpad=15, color='#333333') + ax.set_title('Patients in Training Dataset by Ethnicity', pad=15, color='#333333', + weight='bold') + fig.tight_layout() + + # save fig and clear for next one + plt.savefig("demo_graphs/"+name+"_ethn_graph.png") + plt.clf() + +except: + img = Image.new(mode="RGBA", size=(480,480), color='lightgrey') + draw = ImageDraw.Draw(img) + text = "No Image Available" + draw.text((10,10), text, fill=(0,0,0)) + img.save("demo_graphs/"+name+"_ethn_graph.png") + + +# plot age graph +try: + w = data['train_dataset'][0]['total']['subgroups'][2]['Age'] + y = w.values() + x = w.keys() + + fig, ax = plt.subplots() + fig.set_size_inches(11, 4.8) + bars = ax.bar(x, y, color=(0.06, 0.38, 0.94, 0.7)) + + # formatting graph + ax.spines['top'].set_visible(False) + ax.spines['right'].set_visible(False) + ax.spines['left'].set_visible(False) + ax.spines['bottom'].set_color('#DDDDDD') + + ax.tick_params(bottom=False, left=False) + + ax.set_axisbelow(True) + ax.yaxis.grid(False) + ax.xaxis.grid(False) + + # add text above bars, same color as bars + bar_color = bars[0].get_facecolor() + + for bar in bars: + ax.text( + bar.get_x() + bar.get_width() / 2, + bar.get_height(), + bar.get_height(), + horizontalalignment='center', + color=bar_color, + weight='bold' + ) + + ax.set_xlabel('Age', labelpad=15, color='#333333') + ax.set_ylabel('Number of Patients', labelpad=15, color='#333333') + ax.set_title('Patients in Training Dataset by Age', pad=15, color='#333333', + weight='bold') + fig.tight_layout() + + # save fig + plt.savefig("demo_graphs/"+name+"_age_graph.png") + +except: + img = Image.new(mode="RGBA", size=(480,480), color='lightgrey') + draw = ImageDraw.Draw(img) + text = "No Image Available" + draw.text((10,10), text, fill=(0,0,0)) + img.save("demo_graphs/"+name+"_age_graph.png") -plt.show() \ No newline at end of file