Test/end to end

.github/workflows/publish-coverage.yaml

@@ -4,11 +4,39 @@ on:
   push:
     branches:
       - master
+      - test/end-to-end # for testing purposes to be remoeved later
 
 
 jobs:
-  test:
-    name: Run tests and collect coverage
+  unit-tests:
+    name: Run unit tests and upload coverage
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.24"
+
+      - name: Install dependencies
+        run: go mod download
+
+      - name: Run unit tests with coverage
+        run: go test -coverprofile=coverage.txt -covermode=atomic ./...
+
+      - name: Upload results to Codecov
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./coverage.txt
+          fail_ci_if_error: true
+
+  integration-and-e2e-tests:
+    name: Run integration and e2e tests
     runs-on: ubuntu-latest
     services:
       docker:
@@ -29,6 +57,12 @@ jobs:
           --health-interval=10s
           --health-timeout=5s
           --health-retries=5
+      file-storage:
+        image: seber/maxit-file-storage:latest
+        env:
+          APP_PORT: 8888
+        ports:
+          - 8888:8888
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -43,22 +77,42 @@ jobs:
       - name: Install dependencies
         run: go mod download
 
-      - name: Run unit tests with coverage
-        run: go test -coverprofile=coverage-unit.txt -covermode=atomic ./...
-
-      - name: Run integration tests with coverage
+      - name: Run integration tests
         env:
           DOCKER_HOST: unix:///var/run/docker.sock
-        run: go test -tags=integration -coverprofile=coverage-integration.txt -covermode=atomic ./...
+        run: go test -tags=integration ./...
+
+      - name: Build worker Docker image
+        run: docker build -t worker:test .
 
-      - name: Merge coverage reports
+      - name: Run worker service
+        env:
+          DOCKER_HOST: unix:///var/run/docker.sock
         run: |
-          go install github.com/wadey/gocovmerge@latest
-          gocovmerge coverage-unit.txt coverage-integration.txt > coverage.txt
+          docker run -d \
+            --name worker-service \
+            -v /var/run/docker.sock:/var/run/docker.sock \
+            -e RABBITMQ_HOST=rabbitmq \
+            -e RABBITMQ_PORT=5672 \
+            -e RABBITMQ_USER=guest \
+            -e RABBITMQ_PASSWORD=guest \
+            -e STORAGE_HOST=file-storage \
+            -e STORAGE_PORT=8888 \
+            -e DOCKER_HOST=unix:///var/run/docker.sock \
+            worker:test
 
-      - name: Upload results to Codecov
-        uses: codecov/codecov-action@v5
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          files: ./coverage.txt
-          fail_ci_if_error: true
+      - name: Wait for worker to be ready
+        run: sleep 10
+
+      - name: Run e2e tests
+        env:
+          DOCKER_HOST: unix:///var/run/docker.sock
+        run: go test -tags=e2e ./...
+
+      - name: Show worker logs on failure
+        if: failure()
+        run: docker logs worker-service || docker logs file-storage
+
+      - name: Cleanup worker service
+        if: always()
+        run: docker rm -f worker-service || true

Dockerfile

@@ -1,7 +1,7 @@
 FROM golang:1.23-bookworm AS builder
 
 RUN apt-get update && \
-    apt-get install -y sudo debootstrap schroot g++ && \
+    apt-get install -y sudo debootstrap schroot && \
     rm -rf /var/lib/apt/lists/*
 
 WORKDIR /app
@@ -17,7 +17,6 @@ FROM debian:bookworm-slim
 
 RUN apt-get update && \
 apt-get install -y \
-    g++ \
     docker.io \
     && rm -rf /var/lib/apt/lists/*
 

cmd/main.go

@@ -8,7 +8,6 @@ import (
 	"github.com/mini-maxit/worker/internal/rabbitmq/consumer"
 	"github.com/mini-maxit/worker/internal/rabbitmq/responder"
 	"github.com/mini-maxit/worker/internal/scheduler"
-	"github.com/mini-maxit/worker/internal/stages/compiler"
 	"github.com/mini-maxit/worker/internal/stages/executor"
 	"github.com/mini-maxit/worker/internal/stages/packager"
 	"github.com/mini-maxit/worker/internal/stages/verifier"
@@ -47,7 +46,6 @@ func main() {
 	if err := fileCache.InitCache(); err != nil {
 		logger.Fatalf("Failed to initialize file cache: %v", err)
 	}
-	compiler := compiler.NewCompiler()
 	packager := packager.NewPackager(storageService, fileCache)
 	executor := executor.NewExecutor(dCli)
 	verifier := verifier.NewVerifier(config.VerifierFlags)
@@ -57,7 +55,7 @@ func main() {
 			logger.Error("Failed to close responder", err)
 		}
 	}()
-	scheduler := scheduler.NewScheduler(config.MaxWorkers, compiler, packager, executor, verifier, responder)
+	scheduler := scheduler.NewScheduler(config.MaxWorkers, packager, executor, verifier, responder)
 	consumer := consumer.NewConsumer(workerChannel, config.ConsumeQueueName, scheduler, responder)
 
 	// Start listening for messages

docker-compose.yaml

@@ -1,4 +1,3 @@
-version: '3.8'
 services:
   rabbitmq:
     image: rabbitmq:3.13-management

generate_mocks.sh

@@ -12,7 +12,6 @@ INTERFACES=(
 	"internal/rabbitmq/channel Channel"
 	"internal/rabbitmq/responder Responder"
 	"internal/scheduler Scheduler"
-	"internal/stages/compiler Compiler"
 	"internal/stages/executor Executor"
 	"internal/stages/packager Packager"
 	"internal/stages/verifier Verifier"

internal/docker/docker_client.go

@@ -31,6 +31,7 @@ type DockerClient interface {
 		ctx context.Context,
 		containerID, srcPath, dstPath string,
 		allowedDirs []string,
+		alwaysCopyFiles []string,
 		maxFileSize int64,
 		maxFilesInDir int,
 	) error
@@ -151,6 +152,7 @@ func (d *dockerClient) CopyFromContainerFiltered(
 	ctx context.Context,
 	containerID, srcPath, dstPath string,
 	allowedDirs []string,
+	alwaysCopyFiles []string,
 	maxFileSize int64,
 	maxFilesInDir int,
 ) error {
@@ -163,7 +165,7 @@ func (d *dockerClient) CopyFromContainerFiltered(
 	defer reader.Close()
 
 	// Extract tar archive with filtering to destination
-	err = utils.ExtractTarArchiveFiltered(reader, dstPath, allowedDirs, maxFileSize, maxFilesInDir)
+	err = utils.ExtractTarArchiveFiltered(reader, dstPath, allowedDirs, alwaysCopyFiles, maxFileSize, maxFilesInDir)
 	if err != nil {
 		d.logger.Errorf("Failed to extract filtered tar archive to %s: %s", dstPath, err)
 	}

internal/pipeline/pipeline.go

@@ -1,17 +1,15 @@
 package pipeline
 
 import (
-	"errors"
 	"fmt"
+	"os"
 
 	"github.com/mini-maxit/worker/internal/logger"
 	"github.com/mini-maxit/worker/internal/rabbitmq/responder"
-	"github.com/mini-maxit/worker/internal/stages/compiler"
 	"github.com/mini-maxit/worker/internal/stages/executor"
 	"github.com/mini-maxit/worker/internal/stages/packager"
 	"github.com/mini-maxit/worker/internal/stages/verifier"
 	"github.com/mini-maxit/worker/pkg/constants"
-	customErr "github.com/mini-maxit/worker/pkg/errors"
 	"github.com/mini-maxit/worker/pkg/languages"
 	"github.com/mini-maxit/worker/pkg/messages"
 	"github.com/mini-maxit/worker/pkg/solution"
@@ -37,7 +35,6 @@ type worker struct {
 	id            int
 	state         WorkerState
 	responseQueue string
-	compiler      compiler.Compiler
 	packager      packager.Packager
 	executor      executor.Executor
 	verifier      verifier.Verifier
@@ -47,7 +44,6 @@ type worker struct {
 
 func NewWorker(
 	id int,
-	compiler compiler.Compiler,
 	packager packager.Packager,
 	executor executor.Executor,
 	verifier verifier.Verifier,
@@ -58,7 +54,6 @@ func NewWorker(
 	return &worker{
 		id:        id,
 		state:     WorkerState{Status: constants.WorkerStatusIdle, ProcessingMessageID: ""},
-		compiler:  compiler,
 		packager:  packager,
 		executor:  executor,
 		verifier:  verifier,
@@ -136,28 +131,7 @@ func (ws *worker) ProcessTask(messageID, responseQueue string, task *messages.Ta
 		}
 	}()
 
-	err = ws.compiler.CompileSolutionIfNeeded(
-		langType,
-		task.LanguageVersion,
-		dc.UserSolutionPath,
-		dc.UserExecFilePath,
-		dc.CompileErrFilePath,
-		messageID)
-
-	if err != nil {
-		if errors.Is(err, customErr.ErrCompilationFailed) {
-			ws.publishCompilationError(dc, task.TestCases)
-			return
-		}
-
-		ws.responder.PublishTaskErrorToResponseQueue(
-			constants.QueueMessageTypeTask,
-			ws.state.ProcessingMessageID,
-			ws.responseQueue,
-			err,
-		)
-		return
-	}
+	requiresCompilation := !langType.IsScriptingLanguage()
 
 	limits := make([]solution.Limit, len(task.TestCases))
 	for i, tc := range task.TestCases {
@@ -168,11 +142,13 @@ func (ws *worker) ProcessTask(messageID, responseQueue string, task *messages.Ta
 	}
 
 	cfg := executor.CommandConfig{
-		MessageID:       messageID,
-		DirConfig:       dc,
-		LanguageType:    langType,
-		LanguageVersion: task.LanguageVersion,
-		TestCases:       task.TestCases,
+		MessageID:         messageID,
+		DirConfig:         dc,
+		LanguageType:      langType,
+		LanguageVersion:   task.LanguageVersion,
+		TestCases:         task.TestCases,
+		SourceFilePath:    dc.UserSolutionPath,
+		RequiresCompiling: requiresCompilation,
 	}
 
 	err = ws.executor.ExecuteCommand(cfg)
@@ -186,6 +162,16 @@ func (ws *worker) ProcessTask(messageID, responseQueue string, task *messages.Ta
 		return
 	}
 
+	if requiresCompilation {
+		// Check for compilation error
+		fileInfo, statErr := os.Stat(dc.CompileErrFilePath)
+
+		if statErr == nil && fileInfo.Size() > 0 {
+			ws.publishCompilationError(dc, task.TestCases)
+			return
+		}
+	}
+
 	solutionResult := ws.verifier.EvaluateAllTestCases(dc, task.TestCases, messageID, langType)
 
 	err = ws.packager.SendSolutionPackage(dc, task.TestCases /*hasCompilationErr*/, false, messageID)
@@ -209,6 +195,7 @@ func (ws *worker) ProcessTask(messageID, responseQueue string, task *messages.Ta
 }
 
 func (ws *worker) publishCompilationError(dirConfig *packager.TaskDirConfig, testCases []messages.TestCase) {
+	ws.logger.Infof("Compilation error occurred for message ID: %s", ws.state.ProcessingMessageID)
 	sendErr := ws.packager.SendSolutionPackage(dirConfig, testCases, true, ws.state.ProcessingMessageID)
 	if sendErr != nil {
 		ws.responder.PublishTaskErrorToResponseQueue(