docker-compose.yml

# version: "3.9"
x-airflow-common:
  &airflow-common
  build:
    context: ./bitrix_to_postgres
    dockerfile: Dockerfile.airflow
  environment:
    &airflow-common-env
    AIRFLOW__CORE__EXECUTOR: LocalExecutor
    AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres-2/airflow
    AIRFLOW__CORE__FERNET_KEY: ''
    AIRFLOW__CORE__LOAD_EXAMPLES: 'false'
    AIRFLOW__API__AUTH_BACKENDS: 'airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session'
    AIRFLOW__WEBSERVER__SECRET_KEY: ${AIRFLOW_SECRET_KEY:-your-secret-key-here}
    AIRFLOW__CORE__DEFAULT_TIMEZONE: 'Europe/Moscow'
    AIRFLOW__LOGGING__LOGGING_LEVEL: 'INFO'
    AIRFLOW__SCHEDULER__MAX_TIS_PER_QUERY: '100'
    AIRFLOW__CORE__DAG_FILE_PROCESSOR_TIMEOUT: '600'
    AIRFLOW__CORE__DAGS_FOLDER: '/opt/airflow/dags'
    AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: '30'
    AIRFLOW__CORE__DAG_DISCOVERY_SAFE_MODE: 'false'
    AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS: 'false'
    AIRFLOW__WEBSERVER__EXPOSE_CONFIG: 'true'
    AIRFLOW__CORE__STORE_DAG_CODE: 'true'
    AIRFLOW__CORE__STORE_SERIALIZED_DAGS: 'true'
    PYTHONOPTIMIZE: '1'
    PYTHONPATH: "/opt/airflow"
    AIRFLOW_HOME: "/opt/airflow"
    WEBHOOK: ${WEBHOOK}
    WEBHOOK2: ${WEBHOOK2}
    TZ: 'Europe/Moscow'
    _PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-}
    POSTGRES_USER: ${POSTGRES_USER}
    POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
    POSTGRES_DB: ${POSTGRES_DB}
    AIRFLOW__LOGGING__BASE_LOG_FOLDER: '/opt/airflow/logs'
    AIRFLOW__LOGGING__MAX_BYTES: '5242880'         # 5MB для каждого файла
    AIRFLOW__LOGGING__BACKUP_COUNT: '5'            # Хранить максимум 5 файлов
    AIRFLOW__LOGGING__DELETE_WORKER_LOGS_AFTER: '3'  # Удалять логи воркеров после 3 дней
    AIRFLOW__LOGGING__DAG_FILE_PROCESSOR_MAX_BYTES: '5242880'  # 5MB для процессора DAG
    AIRFLOW__LOGGING__DAG_FILE_PROCESSOR_BACKUP_COUNT: '5'     # Хранить максимум 5 файлов
    AIRFLOW__LOGGING__TASK_LOG_READER: 'task'
    AIRFLOW__LOGGING__DAG_PROCESSOR_MANAGER_LOG_LOCATION: '/opt/airflow/logs/dag_processor_manager/dag_processor_manager.log'
    AIRFLOW__LOGGING__WORKER_LOG_SERVER_PORT: '8793'
    AIRFLOW__LOGGING__COLORED_CONSOLE_LOG: 'false'
    AIRFLOW__LOGGING__LOG_ROTATION_MAX_BYTES: '5242880'  # 5MB
    AIRFLOW__LOGGING__LOG_ROTATION_MAX_DAYS: '7'         # Хранить логи максимум 7 дней

  volumes:
    - ./bitrix_to_postgres/dags:/opt/airflow/dags
    - ./bitrix_to_postgres/logs:/opt/airflow/logs:delegated
    - ./bitrix_to_postgres/plugins:/opt/airflow/plugins
    - ./bitrix_to_postgres:/opt/airflow/bitrix_to_postgres
  user: "${AIRFLOW_UID:-50000}:0"
  depends_on:
    init-airflow-dirs:
      condition: service_completed_successfully
    postgres-2:
      condition: service_healthy

services:
  init-airflow-dirs:
    image: busybox
    networks:
      - airflow-network
    user: "0:0"
    command: >
      sh -c "
        mkdir -p ./bitrix_to_postgres/dags ./bitrix_to_postgres/logs ./bitrix_to_postgres/plugins &&
        chown -R ${AIRFLOW_UID:-50000}:0 ./bitrix_to_postgres/dags ./bitrix_to_postgres/logs ./bitrix_to_postgres/plugins &&
        chmod -R 775 ./bitrix_to_postgres/dags ./bitrix_to_postgres/logs ./bitrix_to_postgres/plugins
      "
    volumes:
      - ./:/workspace
    working_dir: /workspace

  portainer-2:
    image: portainer/portainer-ce
    networks:
      - airflow-network
    ports:
      - 9000:9000
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - portainer_data:/data
    restart: unless-stopped
      
  postgres-2:
    container_name: postgres-2
    image: postgres:16
    networks:
      - airflow-network
    command: postgres -c maintenance_work_mem=64MB -c max_wal_size=1GB -c shared_buffers=256MB -c effective_cache_size=768MB -c work_mem=16MB
    environment:
      POSTGRES_DB: ${POSTGRES_DB}
      POSTGRES_USER: ${POSTGRES_USER}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
      PGDATA: "/var/lib/postgresql/data/pgdata"
    volumes:
      - ../DatabasePOSTGRE:/docker-entrypoint-initdb.d
      - habrdb-data:/var/lib/postgresql/data
    ports:
      - "5432:5432"
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"]
      interval: 10s
      timeout: 5s
      retries: 5
      start_period: 10s
    restart: unless-stopped
    deploy:
      resources:
        limits:
          cpus: '1'
          memory: 4G
  pgadmin-2:
    container_name: pgadmin_container-2
    image: dpage/pgadmin4:latest
    networks:
      - airflow-network
    environment:
      PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL} 
      PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_DEFAULT_PASSWORD}
      PGADMIN_CONFIG_SERVER_MODE: "False"
    volumes:
      - ../pgadmin-data:/var/lib/pgadmin
    ports:
      - "5050:80"
    restart: unless-stopped
    deploy:
      resources:
        limits:
          cpus: '0.5'
          memory: 1G
    

  postgres_exporter-2:
    container_name: exporter_container-2
    image: prometheuscommunity/postgres-exporter:latest
    networks:
      - airflow-network
    environment:
      DATA_SOURCE_URI: "postgres-2:5432/habrdb?sslmode=disable"
      DATA_SOURCE_USER: ${DATA_SOURCE_USER}
      DATA_SOURCE_PASS: ${DATA_SOURCE_PASS}
      PG_EXPORTER_EXTEND_QUERY_PATH: "/etc/postgres_exporter/queries.yaml"
    volumes:
      - ./queries.yaml:/etc/postgres_exporter/queries.yaml:ro
    ports:
      - "9187:9187"
    restart: unless-stopped
    deploy:
      resources:
        limits:
          cpus: '0.2'
          memory: 500M
   
  
   #для личных с папкой
  
  

  bitrix_to_postgres:
    # image: bitrix_to_postgres
    networks:
      - airflow-network
    volumes:
      - ./bitrix_to_postgres:/app

    restart: unless-stopped
    ports:
      - "${PORT_BITRIX_TO_POSTGRES}:${PORT_BITRIX_TO_POSTGRES}"
    environment:
      - POSTGRES_DB=${POSTGRES_DB}
      - POSTGRES_USER=${POSTGRES_USER}
      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
      - POSTGRES_URL=${POSTGRES_URL}
      - WEBHOOK=${WEBHOOK}
  
    
    build:
      context: ./bitrix_to_postgres
    
    depends_on:
      - postgres-2

  airflow-webserver:
    <<: *airflow-common
    networks:
      - airflow-network
    command: airflow webserver
    ports:
      - "8080:8080"
    healthcheck:
      test: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
      interval: 10s
      timeout: 10s
      retries: 5
    restart: always
    deploy:
      resources:
        limits:
          cpus: '1'
          memory: 2G
    depends_on:
      - postgres-2
    volumes:
      - ./bitrix_to_postgres/logs:/opt/airflow/logs:delegated

  airflow-scheduler:
    <<: *airflow-common
    command: airflow scheduler
    healthcheck:
      test: ["CMD-SHELL", 'airflow jobs check --job-type SchedulerJob --hostname "$${HOSTNAME}"']
      interval: 10s
      timeout: 10s
      retries: 5
    restart: always
    networks:
      - airflow-network
    deploy:
      resources:
        limits:
          cpus: '1'
          memory: 2G
    depends_on:
      - postgres-2 
    volumes:
      - ./bitrix_to_postgres/logs:/opt/airflow/logs:delegated

  airflow-init:
    <<: *airflow-common
    entrypoint: /bin/bash
    networks:
      - airflow-network
    command:
      - -c
      - |
        /opt/airflow/bitrix_to_postgres/start-airflow.sh
        airflow db init
        airflow users create \
          --username ${AIRFLOW_USERNAME} \
          --firstname ${AIRFLOW_FIRSTNAME} \
          --lastname ${AIRFLOW_LASTNAME} \
          --role ${AIRFLOW_ROLE} \
          --email ${AIRFLOW_EMAIL} \
          --password ${AIRFLOW_PASSWORD}
    environment:
      <<: *airflow-common-env
      _AIRFLOW_DB_UPGRADE: 'true'
      _AIRFLOW_WWW_USER_CREATE: 'true'

volumes:
  habrdb-data:
  # pgadmin-data:
  portainer_data:

networks:
  airflow-network:
    driver: bridge