Kafka integration

.circleci/config.yml

@@ -1,106 +1,136 @@
 ---
-version: 2
+version: 2.1
+
+parameters:
+  python-version:
+    type: string
+    default: "3.9"
+  python-module:
+    type: string
+    default: "csvapi"
+  publish-branch:
+    type: string
+    default: "master"
+  cache-prefix:
+    type: string
+    default: "py-cache-v1"
 
 jobs:
-  python:
+  install:
     docker:
-      - image: circleci/python:3.7
+      - image: cimg/python:<< pipeline.parameters.python-version >>
     steps:
       - checkout
-      - run:
-          name: Compute Python dependencies key
-          command: cat requirements/*.pip > python.deps
       - run:
           name: Get the base reference branch
           command: export BASE_BRANCH=$(base_branch)
       - restore_cache:
           keys:
-          - py-cache-v2-{{ checksum "python.deps" }}
-          - py-cache-v2-{{ .Branch }}
-          - py-cache-v2-{{ .Environment.BASE_BRANCH }}
+            - << pipeline.parameters.cache-prefix >>-{{ arch }}-{{ checksum "poetry.lock" }}
+            - << pipeline.parameters.cache-prefix >>-{{ arch }}-{{ .Branch }}
+            - << pipeline.parameters.cache-prefix >>-{{ arch }}-{{ .Environment.BASE_BRANCH }}
       - run:
           name: Install python dependencies
           command: |
-            python3 -m venv venv
-            source venv/bin/activate
-            pip install -r requirements/develop.pip || pip install -r requirements/develop.pip
+            poetry config virtualenvs.in-project true
+            poetry install
       - save_cache:
-          key: py-cache-v2-{{ checksum "python.deps" }}
+          key: << pipeline.parameters.cache-prefix >>-{{ arch }}-{{ checksum "poetry.lock" }}
           paths:
-          - venv
+            - .venv
       - save_cache:
-          key: py-cache-v2-{{ .Branch }}
+          key: << pipeline.parameters.cache-prefix >>-{{ arch }}-{{ .Branch }}
+          paths:
+            - .venv
+      - persist_to_workspace:
+          root: .
           paths:
-          - venv
+            - .
+
+  lint:
+    docker:
+      - image: cimg/python:<< pipeline.parameters.python-version >>
+    steps:
+      - attach_workspace:
+          at: .
+      - run:
+          name: Lint code
+          command: poetry run flake8 << pipeline.parameters.python-module >>
+
+  tests:
+    docker:
+      - image: cimg/python:<< pipeline.parameters.python-version >>
+    steps:
+      - attach_workspace:
+          at: .
       - run:
           name: Run tests
           command: |
-            source venv/bin/activate
-            inv qa test --report
+            poetry run pytest --junitxml=reports/python/tests.xml -p no:sugar --color=yes
       - store_test_results:
           path: reports/python
-      - store_artifacts:
-          path: reports/
-          destination: reports
 
-      - persist_to_workspace:
-          root: .
-          paths:
-          - venv
-
-  dist:
+  build:
     docker:
-      - image: circleci/python:3.7
+      - image: cimg/python:<< pipeline.parameters.python-version >>
     steps:
-      - checkout
       - attach_workspace:
           at: .
       - run:
           name: Build a distributable package
           command: |
-            source venv/bin/activate
-            if [[ $CIRCLE_TAG ]]; then inv dist; else inv dist -b $CIRCLE_BUILD_NUM; fi
+            # Build a wheel release
+            if [[ $CIRCLE_TAG ]]; then
+                # This is a tagged release, version has been handled upstream
+                poetry build
+            else
+                # Relies on a dev version like "1.2.1.dev" by default
+                poetry version $(poetry version -s)$CIRCLE_BUILD_NUM
+                poetry build
+            fi
       - store_artifacts:
           path: dist
       - persist_to_workspace:
           root: .
           paths:
-            - dist
+            - .
 
   publish:
     docker:
-      - image: circleci/python:3.7
+      - image: cimg/python:<< pipeline.parameters.python-version >>
     steps:
       - attach_workspace:
           at: .
-      - run:
-          name: Install Twine
-          command: sudo pip install twine
       - deploy:
           name: Publish on PyPI
-          command: twine upload --username "${PYPI_USERNAME}" --password "${PYPI_PASSWORD}" dist/*.whl
+          command: |
+            poetry publish --username "${PYPI_USERNAME}" --password "${PYPI_PASSWORD}" --no-interaction
 
 workflows:
   version: 2
   build:
     jobs:
-      - python:
-          filters:
-            tags:
-              only: /v[0-9]+(\.[0-9]+)*/
-      - dist:
+      - install
+      - lint:
           requires:
-            - python
-          filters:
-            tags:
-              only: /v[0-9]+(\.[0-9]+)*/
+            - install
+      - tests:
+          requires:
+            - install
+      - build:
+          requires:
+            - tests
+            - lint
       - publish:
           requires:
-            - dist
+            - build
           filters:
             branches:
               only:
-                - /v[0-9]+(\.[0-9]+)*/
+                # FIXME: remove when merging on master
+                - kafka-integration
+                - << pipeline.parameters.publish-branch >>
+                - /[0-9]+(\.[0-9]+)+/
             tags:
               only: /v[0-9]+(\.[0-9]+)*/
           context: org-global

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "python.pythonPath": "/Users/geoffrey/.pyenv/versions/3.9.7/envs/csvapi/bin/python",
+    "python.linting.enabled": true,
+    "python.linting.flake8Args": ["--ignore=E24,W504", "--verbose"]
+}

CHANGELOG.md

@@ -2,7 +2,8 @@
 
 ## Current (in progress)
 
-- Nothing yet
+- [BREAKING] Migrate to python >= 3.9
+- Migrate to poetry
 
 ## 1.2.1 (2021-04-29)
 

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2018 Alexandre Bulté
+Copyright 2022 Etalab
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 

README.md

@@ -6,7 +6,7 @@ This tool is used by [data.gouv.fr](https://www.data.gouv.fr) to show a preview
 
 ## Installation
 
-Requires Python 3.7+ and a Unix OS with the `file` command available.
+Requires Python 3.9+ and a Unix OS with the `file` command available.
 
 ```shell
 python3 -m venv pyenv && . pyenv/bin/activate
@@ -16,20 +16,19 @@ pip install csvapi
 For development:
 
 ```shell
-pip install -r requirements/develop.pip
-pip install -e .
+poetry install
 ```
 
 ## Quickstart
 
 ```shell
-csvapi serve -h 0.0.0.0 -p 8000
+poetry run csvapi serve -h 0.0.0.0 -p 8000
 ```
 
 ## Command line options
 
 ```shell
-$ csvapi serve --help
+$ poetry run csvapi serve --help
 Usage: csvapi serve [OPTIONS]
 
 Options:
@@ -245,10 +244,6 @@ By adding `{column}__{comparator}={value}` to the query string, you can filter t
 
 You can add multiple filters, they will be joined with a `AND` at the SQL level.
 
-## Production deployment
-
-Some example [Ansible 2](https://www.ansible.com) roles are [available here](/ansible).
-
 ## Credits
 
 Inspired by the excellent [Datasette](https://github.com/simonw/datasette).