Workflows to use LFS, update dockerfile and dependencies (remove nltk

Author: Peter Johnson

.github/workflows/deploy.yml

@@ -25,6 +25,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
+          lfs: true
 
       - name: Set up Python ${{ matrix.python-version }}
         id: python-setup
@@ -103,4 +104,3 @@ jobs:
       aws-key-id: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_ID }}
       aws-secret-key: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_SECRET}}
       function-admin-api-key: ${{ secrets.FUNCTION_ADMIN_API_KEY}}
-

.github/workflows/production-deploy.yml

@@ -35,6 +35,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
+          lfs: true
 
       - name: Set up Python ${{ matrix.python-version }}
         id: python-setup

.github/workflows/staging-deploy.yml

@@ -25,6 +25,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
+          lfs: true
 
       - name: Set up Python ${{ matrix.python-version }}
         id: python-setup

Dockerfile

@@ -20,17 +20,12 @@ RUN --mount=type=cache,target=$POETRY_CACHE_DIR \
     find /app/.venv -path "*/tests/*" -delete && \
     find /app/.venv -path "*/test/*" -delete && \
     find /app/.venv -name "*.md" -delete && \
-    find /app/.venv -name "*.txt" -delete &&\
-    # Download NLTK corpora into the venv so it's cached ---
-    /app/.venv/bin/python -m nltk.downloader -d /app/.venv/nltk_data brown reuters gutenberg webtext punkt punkt_tab
-
-ENV NLTK_DATA=/app/.venv/nltk_data
+    find /app/.venv -name "*.txt" -delete
 
 FROM ghcr.io/lambda-feedback/evaluation-function-base/python:3.12
 
 ENV VIRTUAL_ENV=/app/.venv \
     PATH="/app/.venv/bin:$PATH"
-    ENV NLTK_DATA=/app/.venv/nltk_data
 
 # Copy the cleaned virtual environment
 COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV}

evaluation_function/models/shannon_words_ngram.py

@@ -8,6 +8,13 @@
 from .utils import csv_to_lists, build_counts
 
 
+import sys, traceback
+def log(msg):
+    sys.stdout.write(msg + "\n")
+    sys.stdout.flush()
+
+log(f"[DEBUG] Starting shannon_words_ngram.py")
+
 # Local users run the following once (no need if using Docker):
 #nltk.download("brown"); nltk.download("reuters"); nltk.download("gutenberg"); nltk.download("webtext")  # CHANGE (one-time)
 

poetry.lock

@@ -26,7 +26,6 @@ typing_extensions = "^4.12.2"
 lf_toolkit = { git = "https://github.com/lambda-feedback/toolkit-python.git", branch = "main", extras = [
     "ipc",
 ] }
-nltk = "^3.9.2"
 numpy = "^2.3.4"
 
 [tool.poetry.group.overrides.dependencies]
@@ -39,6 +38,7 @@ sympy = ">=1.13.3"
 [tool.poetry.group.dev.dependencies]
 pytest = "^8.2.2"
 flake8 = "^7.1.0"
+nltk = "^3.9.2"
 
 [build-system]
 requires = ["poetry-core"]