From 6587408d7c45f94793ee2235ef7cd6891300aca1 Mon Sep 17 00:00:00 2001 From: Eirikur Jonsson Date: Tue, 12 Aug 2025 12:02:44 +0000 Subject: [PATCH 01/24] initial config and azure setup --- azure/azure_style.yaml | 31 +++++++++++++++++++++++++++++- azure/azure_template.yaml | 3 +++ ruff.toml | 40 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 ruff.toml diff --git a/azure/azure_style.yaml b/azure/azure_style.yaml index 5ccf121..e62426d 100644 --- a/azure/azure_style.yaml +++ b/azure/azure_style.yaml @@ -20,7 +20,9 @@ parameters: - name: FPRETTIFY type: boolean default: false - + - name: RUFF + type: boolean + default: false jobs: - job: flake8 pool: @@ -165,3 +167,30 @@ jobs: # Exit with an error if any of the tracked files changed git diff --summary --exit-code + + + - job: ruff + pool: + vmImage: "ubuntu-22.04" + timeoutInMinutes: ${{ parameters.TIMEOUT }} + condition: ${{ parameters.FPRETTIFY }} + steps: + - checkout: self + - checkout: azure_template + - task: UsePythonVersion@0 + inputs: + versionSpec: "3.9" + - script: | + cd ${{ parameters.REPO_NAME }} + pip install ruff + + # Check if we have a local file, if not copy over the global + if [[ ! -f "ruff.toml" ]]; then + cp ../.github/ruff.toml . + fi + + # Run the linting + ruff check . + + # Run the formatting + ruff format . diff --git a/azure/azure_template.yaml b/azure/azure_template.yaml index 20cc52f..d435042 100644 --- a/azure/azure_template.yaml +++ b/azure/azure_template.yaml @@ -54,6 +54,9 @@ parameters: - name: FPRETTIFY type: boolean default: false + - name: RUFF + type: boolean + default: false # Tapenade - name: TAPENADE type: boolean diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..d04ff53 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,40 @@ +# Minimum Python syntax support +target-version = "py39" + +line-length = 120 + +exclude = [ + ".git", + "__pycache__", + "doc/conf.py", + "__init__.py", + "setup.py", +] + +[lint] +ignore = [ + "B006", # do not use mutable data structures for argument defaults (too many false positives) + "B023", # Function definition does not bind loop variable, happens everywhere in our code + "D", # pydocstyle + "E501", # line length, exceeded by some docstrings + "W605", # TODO (to be fixed) # invalid escape sequence, necessary for sphinx directives in docstrings but should switch to raw string +] + +select = [ + "B", + "D", + "E", + "F", + "W", +] + +[lint.pydocstyle] +convention = "numpy" + +[lint.pycodestyle] +max-line-length = 120 + +[format] +quote-style = "double" +indent-style = "space" +docstring-code-format = true From cfb0ec232b54bb6ed3c30acf7e257bb7a773fbe1 Mon Sep 17 00:00:00 2001 From: Eirikur Jonsson Date: Tue, 12 Aug 2025 16:51:30 +0000 Subject: [PATCH 02/24] typo --- azure/azure_style.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure/azure_style.yaml b/azure/azure_style.yaml index e62426d..696b692 100644 --- a/azure/azure_style.yaml +++ b/azure/azure_style.yaml @@ -173,7 +173,7 @@ jobs: pool: vmImage: "ubuntu-22.04" timeoutInMinutes: ${{ parameters.TIMEOUT }} - condition: ${{ parameters.FPRETTIFY }} + condition: ${{ parameters.RUFF }} steps: - checkout: self - checkout: azure_template From cf139a789c34538d210bcccf3e009959e27a4022 Mon Sep 17 00:00:00 2001 From: Alasdair Gray Date: Tue, 12 Aug 2025 16:05:48 -0400 Subject: [PATCH 03/24] Add ruff/pre-commit GHA --- .github/workflows/format-and-lint.yaml | 23 +++++++++++++++++++++++ .pre-commit-config.yaml | 20 ++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 .github/workflows/format-and-lint.yaml create mode 100644 .pre-commit-config.yaml diff --git a/.github/workflows/format-and-lint.yaml b/.github/workflows/format-and-lint.yaml new file mode 100644 index 0000000..0d58432 --- /dev/null +++ b/.github/workflows/format-and-lint.yaml @@ -0,0 +1,23 @@ +on: + workflow_call: + +jobs: + flake8: + runs-on: ubuntu-24.04 + strategy: + matrix: + python-version: [3.9, 3.11] + steps: + - uses: actions/checkout@v5 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + # Change ruffConfig to main before merge + - name: Install dependencies + run: | + wget https://raw.githubusercontent.com/mdolab/.github/ruffConfig/.pre-commit-config.yaml + pip install pre-commit + - name: Run pre-commit checks + run: | + pre-commit run --all-files diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..3d7bc32 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,20 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-json + - id: mixed-line-ending + - id: check-merge-conflict + - id: debug-statements +- repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.12.8 + hooks: + # Run the linter. + - id: ruff-check + args: [ --fix ] + # Run the formatter. + - id: ruff-format From 4e0ef9e649d3fb9c77faf3f0e7b38c9d86b686e4 Mon Sep 17 00:00:00 2001 From: Alasdair Gray Date: Tue, 12 Aug 2025 16:18:37 -0400 Subject: [PATCH 04/24] Fix exclude paths --- ruff.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ruff.toml b/ruff.toml index d04ff53..e82168a 100644 --- a/ruff.toml +++ b/ruff.toml @@ -5,9 +5,9 @@ line-length = 120 exclude = [ ".git", - "__pycache__", + "**/__pycache__", "doc/conf.py", - "__init__.py", + "**/__init__.py", "setup.py", ] From 84bc181ad80df017fbfaa6aeedf26e0ad46f6a8f Mon Sep 17 00:00:00 2001 From: Alasdair Gray Date: Tue, 12 Aug 2025 16:22:56 -0400 Subject: [PATCH 05/24] runpre-commit with --show-diff-on-failure --- .github/workflows/format-and-lint.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/format-and-lint.yaml b/.github/workflows/format-and-lint.yaml index 0d58432..78c0ef1 100644 --- a/.github/workflows/format-and-lint.yaml +++ b/.github/workflows/format-and-lint.yaml @@ -20,4 +20,4 @@ jobs: pip install pre-commit - name: Run pre-commit checks run: | - pre-commit run --all-files + pre-commit run --all-files --show-diff-on-failure From 5944a7dca6edc82e2acc9dab22af16840d96f04f Mon Sep 17 00:00:00 2001 From: Alasdair Gray Date: Tue, 12 Aug 2025 16:41:10 -0400 Subject: [PATCH 06/24] Download ruff config in GHA --- .github/workflows/format-and-lint.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/format-and-lint.yaml b/.github/workflows/format-and-lint.yaml index 78c0ef1..e6a3fd2 100644 --- a/.github/workflows/format-and-lint.yaml +++ b/.github/workflows/format-and-lint.yaml @@ -2,7 +2,7 @@ on: workflow_call: jobs: - flake8: + format-and-lint: runs-on: ubuntu-24.04 strategy: matrix: @@ -17,6 +17,7 @@ jobs: - name: Install dependencies run: | wget https://raw.githubusercontent.com/mdolab/.github/ruffConfig/.pre-commit-config.yaml + wget https://raw.githubusercontent.com/mdolab/.github/ruffConfig/ruff.toml pip install pre-commit - name: Run pre-commit checks run: | From 42bf31a4c37599c85b2e88113d5883f973bf2f79 Mon Sep 17 00:00:00 2001 From: Alasdair Gray Date: Tue, 12 Aug 2025 16:51:22 -0400 Subject: [PATCH 07/24] Don't need to do checks with multiple python versions --- .github/workflows/format-and-lint.yaml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/format-and-lint.yaml b/.github/workflows/format-and-lint.yaml index e6a3fd2..ed3aae8 100644 --- a/.github/workflows/format-and-lint.yaml +++ b/.github/workflows/format-and-lint.yaml @@ -4,15 +4,12 @@ on: jobs: format-and-lint: runs-on: ubuntu-24.04 - strategy: - matrix: - python-version: [3.9, 3.11] steps: - uses: actions/checkout@v5 - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} + python-version: 3.12 # Change ruffConfig to main before merge - name: Install dependencies run: | From a786a416059b1f91ce37b0657432a13728c2b84f Mon Sep 17 00:00:00 2001 From: Alasdair Gray Date: Fri, 15 Aug 2025 17:21:23 -0400 Subject: [PATCH 08/24] More ruff config --- ruff.toml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/ruff.toml b/ruff.toml index e82168a..3c6dde5 100644 --- a/ruff.toml +++ b/ruff.toml @@ -13,11 +13,19 @@ exclude = [ [lint] ignore = [ + "E203", # whitespace before ':', conflicts with black + "E266", # expected 2 blank lines, found 1 + "E302", # do not use mutable data structures for argument defaults (too many false positives) "B006", # do not use mutable data structures for argument defaults (too many false positives) "B023", # Function definition does not bind loop variable, happens everywhere in our code - "D", # pydocstyle "E501", # line length, exceeded by some docstrings - "W605", # TODO (to be fixed) # invalid escape sequence, necessary for sphinx directives in docstrings but should switch to raw string + "W605", # TODO (to be fixed), invalid escape sequence, necessary for sphinx directives in docstrings but should switch to raw string + "D301", # Same reason as W605 + "D10", # D100-107, missing docstring in public function, class etc (yes we should have docstrings for all public methods, but that is a pipe dream) + "D400", # First line of docstring should end with a period (assumes the summary fits on one line, which is not always the case) + "D205", # 1 blank line required between summary line and description (Makes same assumption as D400) + "D200", # One-line docstring should fit on one line, don't like this because it enforces inconsistent formatting between one-line and multi-line docstrings + "D401", # First line of docstring should be in imperative mood, silly rule. ] select = [ @@ -26,15 +34,19 @@ select = [ "E", "F", "W", + "I", ] [lint.pydocstyle] convention = "numpy" -[lint.pycodestyle] -max-line-length = 120 +[lint.isort] +force-sort-within-sections = true [format] quote-style = "double" indent-style = "space" docstring-code-format = true + +[lint.per-file-ignores] +"tests/*" = ["D"] # Ignore docstring checks in tests From f569244204182ac19eb54b6efee02d60efb1e594 Mon Sep 17 00:00:00 2001 From: Alasdair Christison Gray Date: Wed, 20 Aug 2025 15:33:03 -0400 Subject: [PATCH 09/24] More config updates --- ruff.toml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/ruff.toml b/ruff.toml index 3c6dde5..56f7402 100644 --- a/ruff.toml +++ b/ruff.toml @@ -13,11 +13,8 @@ exclude = [ [lint] ignore = [ - "E203", # whitespace before ':', conflicts with black - "E266", # expected 2 blank lines, found 1 - "E302", # do not use mutable data structures for argument defaults (too many false positives) + "E266", # Too many leading '#' for block comment "B006", # do not use mutable data structures for argument defaults (too many false positives) - "B023", # Function definition does not bind loop variable, happens everywhere in our code "E501", # line length, exceeded by some docstrings "W605", # TODO (to be fixed), invalid escape sequence, necessary for sphinx directives in docstrings but should switch to raw string "D301", # Same reason as W605 @@ -34,15 +31,11 @@ select = [ "E", "F", "W", - "I", ] [lint.pydocstyle] convention = "numpy" -[lint.isort] -force-sort-within-sections = true - [format] quote-style = "double" indent-style = "space" From bae7499b69564ca38842f9c4cd97e94d33939703 Mon Sep 17 00:00:00 2001 From: Alasdair Gray Date: Wed, 20 Aug 2025 16:45:33 -0400 Subject: [PATCH 10/24] Enable merging of global and repo-specific ruff config --- .github/workflows/format-and-lint.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/format-and-lint.yaml b/.github/workflows/format-and-lint.yaml index ed3aae8..e5cb7d7 100644 --- a/.github/workflows/format-and-lint.yaml +++ b/.github/workflows/format-and-lint.yaml @@ -14,8 +14,15 @@ jobs: - name: Install dependencies run: | wget https://raw.githubusercontent.com/mdolab/.github/ruffConfig/.pre-commit-config.yaml - wget https://raw.githubusercontent.com/mdolab/.github/ruffConfig/ruff.toml pip install pre-commit + # If there's already a ruff.toml file in the repo, we should rename it to ruff.toml-project, then download the mdolab ruff.toml file edit it to put the line `extend = "ruff.toml-project"` at the top so that ruff will use both configurations. + if [[ -f "ruff.toml" ]]; then + mv ruff.toml ruff.toml-project; + fi + wget https://raw.githubusercontent.com/mdolab/.github/ruffConfig/ruff.toml + if [[ -f "ruff.toml-project" ]]; then + sed -i '1s/^/extend = "ruff.toml-project" \n/' ruff.toml + fi - name: Run pre-commit checks run: | pre-commit run --all-files --show-diff-on-failure From 1147ca4680643b9bdb85f64701461a28d9ecc40d Mon Sep 17 00:00:00 2001 From: Alasdair Gray Date: Wed, 20 Aug 2025 17:08:00 -0400 Subject: [PATCH 11/24] Echo ruff config for verification --- .github/workflows/format-and-lint.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/format-and-lint.yaml b/.github/workflows/format-and-lint.yaml index e5cb7d7..168aec3 100644 --- a/.github/workflows/format-and-lint.yaml +++ b/.github/workflows/format-and-lint.yaml @@ -23,6 +23,12 @@ jobs: if [[ -f "ruff.toml-project" ]]; then sed -i '1s/^/extend = "ruff.toml-project" \n/' ruff.toml fi + echo "Ruff config:" + cat ruff.toml + if [[ -f "ruff.toml-project" ]]; then + echo "Ruff project config:" + cat ruff.toml-project + fi - name: Run pre-commit checks run: | pre-commit run --all-files --show-diff-on-failure From 4e24546da98a7cfca4f67040cba1402a8cc6f186 Mon Sep 17 00:00:00 2001 From: Alasdair Gray Date: Wed, 20 Aug 2025 17:10:35 -0400 Subject: [PATCH 12/24] Run ruff format before ruff check --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3d7bc32..40fc24e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,8 +13,8 @@ repos: # Ruff version. rev: v0.12.8 hooks: + # Run the formatter. + - id: ruff-format # Run the linter. - id: ruff-check args: [ --fix ] - # Run the formatter. - - id: ruff-format From 1aed3c52e148d0195b1467f32c85f5455810e8a1 Mon Sep 17 00:00:00 2001 From: Alasdair Gray Date: Wed, 20 Aug 2025 17:14:11 -0400 Subject: [PATCH 13/24] More rules to ignore --- ruff.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ruff.toml b/ruff.toml index 56f7402..b0bd894 100644 --- a/ruff.toml +++ b/ruff.toml @@ -22,7 +22,8 @@ ignore = [ "D400", # First line of docstring should end with a period (assumes the summary fits on one line, which is not always the case) "D205", # 1 blank line required between summary line and description (Makes same assumption as D400) "D200", # One-line docstring should fit on one line, don't like this because it enforces inconsistent formatting between one-line and multi-line docstrings - "D401", # First line of docstring should be in imperative mood, silly rule. + "D202", # No blank lines allowed after function docstring (silly rule, if formatter doesn't enforce it then checks shouldn't either) + "D401", # First line of docstring should be in imperative mood (silly rule) ] select = [ From 7d639d3b17b9834fb12f5ff2a83f003cafb6b594 Mon Sep 17 00:00:00 2001 From: Alasdair Gray Date: Wed, 20 Aug 2025 17:18:28 -0400 Subject: [PATCH 14/24] Nuke flake8 and black from the repo --- .flake8 | 59 -------------------------------- .github/PULL_REQUEST_TEMPLATE.md | 2 +- .github/workflows/black.yaml | 17 --------- .github/workflows/flake8.yaml | 34 ------------------ README.md | 2 +- azure/README.md | 46 ++++++++++++------------- azure/azure_style.yaml | 54 +++-------------------------- flake8-requirements.txt | 7 ---- 8 files changed, 30 insertions(+), 191 deletions(-) delete mode 100644 .flake8 delete mode 100644 .github/workflows/black.yaml delete mode 100644 .github/workflows/flake8.yaml delete mode 100644 flake8-requirements.txt diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 101bcb4..0000000 --- a/.flake8 +++ /dev/null @@ -1,59 +0,0 @@ -[flake8] -max-line-length = 120 - -extend-ignore = - # black - E203, - # too many leading '#' for block comment - E266, - # expected 2 blank lines, found 1 - E302, - # do not use mutable data structures for argument defaults (too many false positives) - B006, - # ===== TODO: to be fixed: - # invalid escape sequence, necessary for sphinx directives in docstrings but should switch to raw string - W605, - # line length, exceeded by some docstrings - E501, - # Function definition does not bind loop variable, happens everywhere in our code - B023, - # pydocstyle - D - -# Only add patterns here that are not included by the defaults of flake8 or other plugins -# extend-select = - -# flake8-docstrings -docstring-convention = numpy - -# flake8-rst-docstrings -rst-roles = - class, - func, - ref, - meth, - -rst-directives = - # Custom directives defined in the sphinx_mdolab_theme - embed-compare, - embed-bibtex, - embed-code, - embed-shell-cmd, - embed-n2, - -# mccabe complexity -# max-complexity = 10 - -# ignored files/directories -# we use exclude here and extend-exclude in repo-specific config files -# so that we can pass both to flake8 directly without needing to merge them first -exclude = - # No need to traverse the git directory - .git, - # There's no value in checking cache directories - __pycache__, - # The conf file is mostly autogenerated, ignore it - doc/conf.py, - # No need for init and setup files - __init__.py, - setup.py, diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index d7caabd..5a506c4 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -32,7 +32,7 @@ Select the appropriate type(s) that describe this PR ## Checklist -- [ ] I have run `flake8` and `black` to make sure the Python code adheres to PEP-8 and is consistently formatted +- [ ] I have run `ruff check` and `ruff format` to make sure the Python code adheres to PEP-8 and is consistently formatted - [ ] I have formatted the Fortran code with `fprettify` or C/C++ code with `clang-format` as applicable - [ ] I have run unit and regression tests which pass locally with my changes - [ ] I have added new tests that prove my fix is effective or that my feature works diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml deleted file mode 100644 index b87347b..0000000 --- a/.github/workflows/black.yaml +++ /dev/null @@ -1,17 +0,0 @@ -on: - workflow_call: - inputs: - black_version: - required: false - type: string - default: "23.1.0" - -jobs: - black: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - - uses: psf/black@stable - with: - options: "--check --diff -l 120 --target-version py39 --target-version py311" - version: ${{ inputs.black_version }} diff --git a/.github/workflows/flake8.yaml b/.github/workflows/flake8.yaml deleted file mode 100644 index e03de52..0000000 --- a/.github/workflows/flake8.yaml +++ /dev/null @@ -1,34 +0,0 @@ -on: - workflow_call: - -jobs: - flake8: - runs-on: ubuntu-24.04 - strategy: - matrix: - python-version: [3.9, 3.11] - steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - pip install wheel - wget https://raw.githubusercontent.com/mdolab/.github/main/flake8-requirements.txt - pip install -r flake8-requirements.txt - - name: Lint with flake8 - run: | - if [[ -f ".flake8" ]]; then - export FL8=.flake8-project - mv .flake8 $FL8; # rename the file from code repo; should have higher precedence in merge - fi - - wget https://raw.githubusercontent.com/mdolab/.github/main/.flake8; - - if [[ -f "$FL8" ]]; then - flake8 . --append-config $FL8 --count --show-source --statistics; - else - flake8 . --count --show-source --statistics; - fi diff --git a/README.md b/README.md index 55ff7e4..2c91e64 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ This repo stores the following shared repository settings/configurations/templates: - Issue and pull request templates on GitHub - Shared configurations for - - `flake8` + - `ruff` - `isort` - `pylint` - `codecov` diff --git a/azure/README.md b/azure/README.md index b2b8a57..0a4d276 100644 --- a/azure/README.md +++ b/azure/README.md @@ -10,29 +10,29 @@ The templates are organized into the following files: - `azure_template.yaml` which handles the job itself, calling the necessary sub-templates. ## Template Options -| Name | Type | Default | Description. | -| :------------------ | :------ | :--------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------ | -| `REPO_NAME` | string | | Name of repository | -| `COMPLEX` | boolean | `false` | Flag for triggering complex build and tests | -| `TIMEOUT_BUILD` | number | `120` | Runtime allowed for each build and test job, in minutes | -| `GCC_CONFIG` | string | `None` | Path to GCC configuration file (from repository root) | -| `INTEL_CONFIG` | string | `None` | Path to Intel configuration file (from repository root) | -| `BUILD_REAL` | string | `.github/build_real.sh` | Path to Bash script with commands to build real code. Using `None` will skip this step. | -| `TEST_REAL` | string | `.github/text_real.sh` | Path to Bash script to run real tests. Using `None` will skip this step. | -| `BUILD_COMPLEX` | string | `.github/build_complex.sh` | Path to Bash script with commands to build complex code. Using `None` will skip this step. | -| `TEST_COMPLEX` | string | `.github/text_complex.sh` | Path to Bash script with commands to run complex tests. Using `None` will skip this step. | -| `IMAGE` | string | `public` | Select Docker image. Can be `public`, `private`, or `auto`. `auto` uses the private image on trusted builds and the public image otherwise. | -| `COVERAGE` | boolean | `false` | Flag to report test coverage to `codecov` | -| `TIMEOUT_STYLE` | number | `10` | Runtime allowed for each style check, in minutes | -| `IGNORE_STYLE` | boolean | `false` | Flag to allow `black` and `flake8` checks to fail without failing the pipeline | -| `ISORT` | boolean | `false` | Flag to trigger the `isort` check | -| `PYLINT` | boolean | `false` | Flag to trigger the `pylint` check | -| `CLANG_FORMAT` | boolean | `false` | Flag to trigger the `clang-format` check | -| `FPRETTIFY` | boolean | `false` | Flag to trigger the `fprettify` check | -| `TAPENADE` | boolean | `false` | Flag to trigger the Tapenade check | -| `TIMEOUT_TAPENADE` | number | `10` | Runtime allowed for the Tapenade check, in minutes | -| `TAPENADE_SCRIPT` | string | `.github/build_tapenade.sh` | Path to Bash script with commands to run Tapenade | -| `TAPENADE_VERSION` | string | `3.10` | Version of Tapenade to use | +| Name | Type | Default | Description. | +| :----------------- | :------ | :-------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------ | +| `REPO_NAME` | string | | Name of repository | +| `COMPLEX` | boolean | `false` | Flag for triggering complex build and tests | +| `TIMEOUT_BUILD` | number | `120` | Runtime allowed for each build and test job, in minutes | +| `GCC_CONFIG` | string | `None` | Path to GCC configuration file (from repository root) | +| `INTEL_CONFIG` | string | `None` | Path to Intel configuration file (from repository root) | +| `BUILD_REAL` | string | `.github/build_real.sh` | Path to Bash script with commands to build real code. Using `None` will skip this step. | +| `TEST_REAL` | string | `.github/text_real.sh` | Path to Bash script to run real tests. Using `None` will skip this step. | +| `BUILD_COMPLEX` | string | `.github/build_complex.sh` | Path to Bash script with commands to build complex code. Using `None` will skip this step. | +| `TEST_COMPLEX` | string | `.github/text_complex.sh` | Path to Bash script with commands to run complex tests. Using `None` will skip this step. | +| `IMAGE` | string | `public` | Select Docker image. Can be `public`, `private`, or `auto`. `auto` uses the private image on trusted builds and the public image otherwise. | +| `COVERAGE` | boolean | `false` | Flag to report test coverage to `codecov` | +| `TIMEOUT_STYLE` | number | `10` | Runtime allowed for each style check, in minutes | +| `IGNORE_STYLE` | boolean | `false` | Flag to allow `formatting and linting` checks to fail without failing the pipeline | +| `ISORT` | boolean | `false` | Flag to trigger the `isort` check | +| `PYLINT` | boolean | `false` | Flag to trigger the `pylint` check | +| `CLANG_FORMAT` | boolean | `false` | Flag to trigger the `clang-format` check | +| `FPRETTIFY` | boolean | `false` | Flag to trigger the `fprettify` check | +| `TAPENADE` | boolean | `false` | Flag to trigger the Tapenade check | +| `TIMEOUT_TAPENADE` | number | `10` | Runtime allowed for the Tapenade check, in minutes | +| `TAPENADE_SCRIPT` | string | `.github/build_tapenade.sh` | Path to Bash script with commands to run Tapenade | +| `TAPENADE_VERSION` | string | `3.10` | Version of Tapenade to use | ## Setting up a pipeline ### Step 1: Setup Azure Pipelines YAML File: diff --git a/azure/azure_style.yaml b/azure/azure_style.yaml index 696b692..7dfa2bf 100644 --- a/azure/azure_style.yaml +++ b/azure/azure_style.yaml @@ -24,59 +24,11 @@ parameters: type: boolean default: false jobs: - - job: flake8 - pool: - vmImage: "ubuntu-22.04" - timeoutInMinutes: ${{ parameters.TIMEOUT }} - continueOnError: ${{ parameters.IGNORE_STYLE }} - strategy: - matrix: - "py39": - PYTHON_VERSION: "3.9" - "py311": - PYTHON_VERSION: "3.11" - steps: - - checkout: self - - checkout: azure_template - - task: UsePythonVersion@0 - inputs: - versionSpec: $(PYTHON_VERSION) - - script: | - cd ${{ parameters.REPO_NAME }} - pip install wheel - pip install -r ../.github/flake8-requirements.txt - - if [[ -f ".flake8" ]]; then - export FL8=.flake8-project - mv .flake8 $FL8; # rename the file from code repo; should have higher precedence in merge - fi - - cp ../.github/.flake8 . - - if [[ -f "$FL8" ]]; then - flake8 . --append-config $FL8 --count --show-source --statistics; - else - flake8 . --count --show-source --statistics; - fi - - - job: black - pool: - vmImage: "ubuntu-22.04" - timeoutInMinutes: ${{ parameters.TIMEOUT }} - continueOnError: ${{ parameters.IGNORE_STYLE }} - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: "3.9" - - script: | - pip install wheel - pip install black==23.1.0 - black . --check --diff -l 120 --target-version py39 --target-version py311 - - job: isort pool: vmImage: "ubuntu-22.04" timeoutInMinutes: ${{ parameters.TIMEOUT }} + continueOnError: ${{ parameters.IGNORE_STYLE }} condition: ${{ parameters.ISORT }} steps: - checkout: self @@ -100,6 +52,7 @@ jobs: pool: vmImage: "ubuntu-22.04" timeoutInMinutes: ${{ parameters.TIMEOUT }} + continueOnError: ${{ parameters.IGNORE_STYLE }} condition: ${{ parameters.PYLINT }} steps: - checkout: self @@ -120,6 +73,7 @@ jobs: pool: vmImage: "ubuntu-22.04" timeoutInMinutes: ${{ parameters.TIMEOUT }} + continueOnError: ${{ parameters.IGNORE_STYLE }} condition: ${{ parameters.CLANG_FORMAT }} steps: - checkout: self @@ -143,6 +97,7 @@ jobs: pool: vmImage: "ubuntu-22.04" timeoutInMinutes: ${{ parameters.TIMEOUT }} + continueOnError: ${{ parameters.IGNORE_STYLE }} condition: ${{ parameters.FPRETTIFY }} steps: - checkout: self @@ -173,6 +128,7 @@ jobs: pool: vmImage: "ubuntu-22.04" timeoutInMinutes: ${{ parameters.TIMEOUT }} + continueOnError: ${{ parameters.IGNORE_STYLE }} condition: ${{ parameters.RUFF }} steps: - checkout: self diff --git a/flake8-requirements.txt b/flake8-requirements.txt deleted file mode 100644 index 6be24e5..0000000 --- a/flake8-requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -flake8==5.0.4 -flake8-bugbear -flake8-builtins -flake8-docstrings -flake8-rst-docstrings -# flake8-comprehensions -# flake8-broken-line From d17cc8a014854e34fef8e2e3ebbc8fcb25b10e4e Mon Sep 17 00:00:00 2001 From: Alasdair Gray Date: Thu, 21 Aug 2025 15:20:14 -0400 Subject: [PATCH 15/24] Update ruff version --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 40fc24e..1632682 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: - id: debug-statements - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.12.8 + rev: v0.12.10 hooks: # Run the formatter. - id: ruff-format From b422d2d72772733d3b280a71159ef7778e17dc99 Mon Sep 17 00:00:00 2001 From: Alasdair Gray Date: Thu, 21 Aug 2025 16:19:13 -0400 Subject: [PATCH 16/24] Ignore D404 --- ruff.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/ruff.toml b/ruff.toml index b0bd894..ac85386 100644 --- a/ruff.toml +++ b/ruff.toml @@ -24,6 +24,7 @@ ignore = [ "D200", # One-line docstring should fit on one line, don't like this because it enforces inconsistent formatting between one-line and multi-line docstrings "D202", # No blank lines allowed after function docstring (silly rule, if formatter doesn't enforce it then checks shouldn't either) "D401", # First line of docstring should be in imperative mood (silly rule) + "D404", # First word of the docstring should not be "This" (Probably a good rule to follow going forward but it occurs in so many places in our code and it's not a big enough issue to warrant fixing) ] select = [ From fe87ae48b5ea1cd268f76d97cfcc67707b136fca Mon Sep 17 00:00:00 2001 From: Alasdair Gray Date: Thu, 21 Aug 2025 20:49:42 -0400 Subject: [PATCH 17/24] Change precedence of ruff configs --- .github/workflows/format-and-lint.yaml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/format-and-lint.yaml b/.github/workflows/format-and-lint.yaml index 168aec3..ddc95a6 100644 --- a/.github/workflows/format-and-lint.yaml +++ b/.github/workflows/format-and-lint.yaml @@ -15,20 +15,19 @@ jobs: run: | wget https://raw.githubusercontent.com/mdolab/.github/ruffConfig/.pre-commit-config.yaml pip install pre-commit - # If there's already a ruff.toml file in the repo, we should rename it to ruff.toml-project, then download the mdolab ruff.toml file edit it to put the line `extend = "ruff.toml-project"` at the top so that ruff will use both configurations. + # If there's already a ruff.toml file in the repo, we should rename the file we're downloading to ruff.toml-global, then put the line `extend = "ruff.toml-global"` at the top of the local ruff.toml so that ruff will use both configurations. + url=https://raw.githubusercontent.com/mdolab/.github/ruffConfig/ruff.toml if [[ -f "ruff.toml" ]]; then - mv ruff.toml ruff.toml-project; - fi - wget https://raw.githubusercontent.com/mdolab/.github/ruffConfig/ruff.toml - if [[ -f "ruff.toml-project" ]]; then - sed -i '1s/^/extend = "ruff.toml-project" \n/' ruff.toml + wget $url -O ruff.toml-global; + sed -i '1s/^/extend = "ruff.toml-global" \n/' ruff.toml + else + wget $url fi echo "Ruff config:" - cat ruff.toml - if [[ -f "ruff.toml-project" ]]; then - echo "Ruff project config:" - cat ruff.toml-project + if [[ -f "ruff.toml-global" ]]; then + cat ruff.toml-global fi + cat ruff.toml - name: Run pre-commit checks run: | pre-commit run --all-files --show-diff-on-failure From fcd2e936d6fc7eca745f65e05df672632ac8b553 Mon Sep 17 00:00:00 2001 From: Alasdair Gray Date: Thu, 21 Aug 2025 20:58:42 -0400 Subject: [PATCH 18/24] Update .pre-commit-config.yaml --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1632682..36a9353 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,8 +13,8 @@ repos: # Ruff version. rev: v0.12.10 hooks: - # Run the formatter. - - id: ruff-format # Run the linter. - id: ruff-check - args: [ --fix ] + args: [ --fix, --exit-non-zero-on-fix ] + # Run the formatter. + - id: ruff-format From c0c2e18fd74e961ea2c8980057b7f945180d3e7c Mon Sep 17 00:00:00 2001 From: Alasdair Gray Date: Thu, 21 Aug 2025 21:08:35 -0400 Subject: [PATCH 19/24] Remove Azure Ruff check --- azure/azure_style.yaml | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/azure/azure_style.yaml b/azure/azure_style.yaml index 7dfa2bf..4137a34 100644 --- a/azure/azure_style.yaml +++ b/azure/azure_style.yaml @@ -20,9 +20,6 @@ parameters: - name: FPRETTIFY type: boolean default: false - - name: RUFF - type: boolean - default: false jobs: - job: isort pool: @@ -122,31 +119,3 @@ jobs: # Exit with an error if any of the tracked files changed git diff --summary --exit-code - - - - job: ruff - pool: - vmImage: "ubuntu-22.04" - timeoutInMinutes: ${{ parameters.TIMEOUT }} - continueOnError: ${{ parameters.IGNORE_STYLE }} - condition: ${{ parameters.RUFF }} - steps: - - checkout: self - - checkout: azure_template - - task: UsePythonVersion@0 - inputs: - versionSpec: "3.9" - - script: | - cd ${{ parameters.REPO_NAME }} - pip install ruff - - # Check if we have a local file, if not copy over the global - if [[ ! -f "ruff.toml" ]]; then - cp ../.github/ruff.toml . - fi - - # Run the linting - ruff check . - - # Run the formatting - ruff format . From fe2dbee1f436770df3e93233ec7ef79ed5dfd196 Mon Sep 17 00:00:00 2001 From: Alasdair Gray Date: Thu, 21 Aug 2025 21:09:19 -0400 Subject: [PATCH 20/24] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 2c91e64..9c726c0 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ This repo stores the following shared repository settings/configurations/templates: - Issue and pull request templates on GitHub - Shared configurations for + - `pre-commit` - `ruff` - `isort` - `pylint` From 119b9d85098f32f2b638130cd768cfbfa08940bd Mon Sep 17 00:00:00 2001 From: Alasdair Gray Date: Fri, 22 Aug 2025 09:16:48 -0400 Subject: [PATCH 21/24] Revert "Remove Azure Ruff check" This reverts commit c0c2e18fd74e961ea2c8980057b7f945180d3e7c. --- azure/azure_style.yaml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/azure/azure_style.yaml b/azure/azure_style.yaml index 4137a34..7dfa2bf 100644 --- a/azure/azure_style.yaml +++ b/azure/azure_style.yaml @@ -20,6 +20,9 @@ parameters: - name: FPRETTIFY type: boolean default: false + - name: RUFF + type: boolean + default: false jobs: - job: isort pool: @@ -119,3 +122,31 @@ jobs: # Exit with an error if any of the tracked files changed git diff --summary --exit-code + + + - job: ruff + pool: + vmImage: "ubuntu-22.04" + timeoutInMinutes: ${{ parameters.TIMEOUT }} + continueOnError: ${{ parameters.IGNORE_STYLE }} + condition: ${{ parameters.RUFF }} + steps: + - checkout: self + - checkout: azure_template + - task: UsePythonVersion@0 + inputs: + versionSpec: "3.9" + - script: | + cd ${{ parameters.REPO_NAME }} + pip install ruff + + # Check if we have a local file, if not copy over the global + if [[ ! -f "ruff.toml" ]]; then + cp ../.github/ruff.toml . + fi + + # Run the linting + ruff check . + + # Run the formatting + ruff format . From cb9f19636764a53c9136be596a362c4cd850affd Mon Sep 17 00:00:00 2001 From: Alasdair Gray Date: Fri, 22 Aug 2025 09:25:09 -0400 Subject: [PATCH 22/24] Update azure ruff check --- azure/README.md | 1 + azure/azure_style.yaml | 68 ++++++++++++++++++++++++------------------ 2 files changed, 40 insertions(+), 29 deletions(-) diff --git a/azure/README.md b/azure/README.md index 0a4d276..3b6737e 100644 --- a/azure/README.md +++ b/azure/README.md @@ -25,6 +25,7 @@ The templates are organized into the following files: | `COVERAGE` | boolean | `false` | Flag to report test coverage to `codecov` | | `TIMEOUT_STYLE` | number | `10` | Runtime allowed for each style check, in minutes | | `IGNORE_STYLE` | boolean | `false` | Flag to allow `formatting and linting` checks to fail without failing the pipeline | +| `RUFF` | boolean | `false` | Flag to trigger the `ruff` check | | `ISORT` | boolean | `false` | Flag to trigger the `isort` check | | `PYLINT` | boolean | `false` | Flag to trigger the `pylint` check | | `CLANG_FORMAT` | boolean | `false` | Flag to trigger the `clang-format` check | diff --git a/azure/azure_style.yaml b/azure/azure_style.yaml index 7dfa2bf..0861436 100644 --- a/azure/azure_style.yaml +++ b/azure/azure_style.yaml @@ -8,6 +8,9 @@ parameters: - name: IGNORE_STYLE type: boolean default: false + - name: RUFF + type: boolean + default: true - name: ISORT type: boolean default: false @@ -20,10 +23,43 @@ parameters: - name: FPRETTIFY type: boolean default: false - - name: RUFF - type: boolean - default: false jobs: + - job: ruff + pool: + vmImage: "ubuntu-22.04" + timeoutInMinutes: ${{ parameters.TIMEOUT }} + continueOnError: ${{ parameters.IGNORE_STYLE }} + condition: ${{ parameters.RUFF }} + steps: + - checkout: self + - checkout: azure_template + - task: UsePythonVersion@0 + inputs: + versionSpec: "3.9" + - script: | + cd ${{ parameters.REPO_NAME }} + pip install ruff==0.12.10 + + # If there's already a ruff.toml file in the repo, we should rename the global config to ruff.toml-global, then put the line `extend = "ruff.toml-global"` at the top of the local ruff.toml so that ruff will use both configurations. + globalConfigFile="../.github/ruff.toml" + if [[ -f "ruff.toml" ]]; then + cp $globalConfigFile ruff.toml-global + sed -i '1s/^/extend = "ruff.toml-global" \n/' ruff.toml + else + cp $globalConfigFile . + fi + echo "Ruff config:" + if [[ -f "ruff.toml-global" ]]; then + cat ruff.toml-global + fi + cat ruff.toml + + # Run the linting + ruff check . + + # Run the formatting + ruff format . + - job: isort pool: vmImage: "ubuntu-22.04" @@ -124,29 +160,3 @@ jobs: git diff --summary --exit-code - - job: ruff - pool: - vmImage: "ubuntu-22.04" - timeoutInMinutes: ${{ parameters.TIMEOUT }} - continueOnError: ${{ parameters.IGNORE_STYLE }} - condition: ${{ parameters.RUFF }} - steps: - - checkout: self - - checkout: azure_template - - task: UsePythonVersion@0 - inputs: - versionSpec: "3.9" - - script: | - cd ${{ parameters.REPO_NAME }} - pip install ruff - - # Check if we have a local file, if not copy over the global - if [[ ! -f "ruff.toml" ]]; then - cp ../.github/ruff.toml . - fi - - # Run the linting - ruff check . - - # Run the formatting - ruff format . From 0fda72f21335dd47b20f6fd3317d162bc289e818 Mon Sep 17 00:00:00 2001 From: Alasdair Gray Date: Fri, 22 Aug 2025 09:25:50 -0400 Subject: [PATCH 23/24] Use `--check` with `ruff format` --- azure/azure_style.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure/azure_style.yaml b/azure/azure_style.yaml index 0861436..bb0e8f4 100644 --- a/azure/azure_style.yaml +++ b/azure/azure_style.yaml @@ -58,7 +58,7 @@ jobs: ruff check . # Run the formatting - ruff format . + ruff format --check . - job: isort pool: From 560863f021db62c9eb67bc9497583ac813590914 Mon Sep 17 00:00:00 2001 From: Alasdair Gray Date: Fri, 22 Aug 2025 09:27:15 -0400 Subject: [PATCH 24/24] Fix branch that GHA pulls ruff config from --- .github/workflows/format-and-lint.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/format-and-lint.yaml b/.github/workflows/format-and-lint.yaml index ddc95a6..cfd9a4c 100644 --- a/.github/workflows/format-and-lint.yaml +++ b/.github/workflows/format-and-lint.yaml @@ -10,18 +10,17 @@ jobs: uses: actions/setup-python@v5 with: python-version: 3.12 - # Change ruffConfig to main before merge - name: Install dependencies run: | - wget https://raw.githubusercontent.com/mdolab/.github/ruffConfig/.pre-commit-config.yaml + wget https://raw.githubusercontent.com/mdolab/.github/main/.pre-commit-config.yaml pip install pre-commit # If there's already a ruff.toml file in the repo, we should rename the file we're downloading to ruff.toml-global, then put the line `extend = "ruff.toml-global"` at the top of the local ruff.toml so that ruff will use both configurations. - url=https://raw.githubusercontent.com/mdolab/.github/ruffConfig/ruff.toml + url=https://raw.githubusercontent.com/mdolab/.github/main/ruff.toml if [[ -f "ruff.toml" ]]; then wget $url -O ruff.toml-global; sed -i '1s/^/extend = "ruff.toml-global" \n/' ruff.toml else - wget $url + wget $url fi echo "Ruff config:" if [[ -f "ruff.toml-global" ]]; then