docker-compose.yml

version: '3.7'
networks:
  awari-pipelines:
      driver: bridge
      ipam:
          driver: default
          config:
              - subnet: "172.18.0.0/16"
                gateway: "172.18.0.1"

# Settings and configurations that are common for all minio containers
x-minio-common: &minio-common
  image: minio/minio:RELEASE.2023-02-22T18-23-45Z
  command: server --console-address ":9001" http://minio{1...4}/data{1...2}
  expose:
    - "9000"
    - "9001"
  environment:
    MINIO_ROOT_USER: minioadmin
    MINIO_ROOT_PASSWORD: minioadmin
  healthcheck:
    test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
    interval: 30s
    timeout: 20s
    retries: 3

# Settings dna configuration for apache-airflow
x-awari-airflow-common: &awari-airflow-common
  build: 
    dockerfile: data/airflow/Dockerfile
  # image: apache/airflow:2.5.1-python3.8
  user: "${AIRFLOW_UID}:0"
  env_file: 
    - data/airflow/.env
  volumes:
    - ./data/airflow/dags:/opt/airflow/dags
    - ./data/airflow/logs:/opt/airflow/logs
    - ./data/airflow/downloads:/opt/airflow/downloads
    - ./data/airflow/plugins:/opt/airflow/plugins
    - /var/run/docker.sock:/var/run/docker.sock

x-awari-airflow-depends-on: &awari-airflow-depends-on
  depends_on:
    awari-airflow-postgres:
      condition: service_healthy
    awari-airflow-init:
      condition: service_completed_successfully

# starts 4 docker containers running minio server instances.
# using nginx reverse proxy, load balancing, you can access
# it through port 9000.
services:
  minio1:
    <<: *minio-common
    hostname: minio1
    volumes:
      - ./data/minio/data1-1:/data1
      - ./data/minio/data1-2:/data2
    networks:
      awari-pipelines:
        ipv4_address: 172.18.0.16

  minio2:
    <<: *minio-common
    hostname: minio2
    volumes:
      - ./data/minio/data2-1:/data1
      - ./data/minio/data2-2:/data2
    networks:
      awari-pipelines:
        ipv4_address: 172.18.0.17

  minio3:
    <<: *minio-common
    hostname: minio3
    volumes:
      - ./data/minio/data3-1:/data1
      - ./data/minio/data3-2:/data2
    networks:
      awari-pipelines:
        ipv4_address: 172.18.0.18

  minio4:
    <<: *minio-common
    hostname: minio4
    volumes:
      - ./data/minio/data4-1:/data1
      - ./data/minio/data4-2:/data2
    networks:
      awari-pipelines:
        ipv4_address: 172.18.0.19

  awari-minio-nginx:
    image: nginx:1.19.2-alpine
    hostname: awari-nginx
    volumes:
      - ./data/minio/nginx/nginx.conf:/etc/nginx/nginx.conf:ro
    ports:
      - "9000:9000"
      - "9001:9001"
    depends_on:
      - minio1
      - minio2
      - minio3
      - minio4
    networks:
      awari-pipelines:
        ipv4_address: 172.18.0.20

  awari-mongodb:
    image: mongo:latest
    environment:
      MONGO_INITDB_ROOT_USERNAME: root
      MONGO_INITDB_ROOT_PASSWORD: rootpassword
    ports:
      - 27017:27017
    volumes:
      - ./data/mongodb/data:/data/db
    networks:
      awari-pipelines:
        ipv4_address: 172.18.0.21

  # Python service
  python-app:
    build:
      dockerfile: data/python/Dockerfile
    volumes:
      - ./python-app:/home/awari/app
    ports:
      - "5001:5001"
    networks:
      - awari-pipelines

  # Airflow Services
  awari-airflow-postgres:
    image: postgres:latest
    container_name: awari-airflow-postgres
    ports:
      - "5434:5432"
    healthcheck:
      test: ["CMD", "pg_isready", "-U", "airflow"]
      interval: 5s
      retries: 5
    env_file:
      - data/airflow/.env
    volumes:
      - ./data/airflow/postgres-data:/var/lib/postgresql/data
    networks:
      awari-pipelines:
        ipv4_address: 172.18.0.110

  awari-airflow-scheduler:
    <<: *awari-airflow-common
    <<: *awari-airflow-depends-on
    container_name: awari-airflow-scheduler
    command: scheduler
    restart: on-failure
    ports:
      - "8793:8793"
    networks:
      awari-pipelines:
        ipv4_address: 172.18.0.111

  awari-airflow-webserver:
    <<: *awari-airflow-common
    <<: *awari-airflow-depends-on
    container_name: awari-airflow-webserver
    restart: always
    command: webserver
    ports:
      - "8080:8080"
    healthcheck:
      test: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
      interval: 30s
      timeout: 30s
      retries: 5
    networks:
        awari-pipelines:
          ipv4_address: 172.18.0.112
  
  awari-airflow-init:
    <<: *awari-airflow-common
    container_name: awari-airflow-init
    entrypoint: /bin/bash
    command:
      - -c
      - |
        mkdir -p /sources/logs /sources/dags /sources/plugins /sources/downloads
        chmod 777 -R /sources/downloads
        chown -R "${AIRFLOW_UID}:0" /sources/{logs,dags,plugins,downloads}
        exec /entrypoint airflow version
    networks:
      - awari-pipelines

  # Postgreql Service
  awari-postgres:
    image: postgres:15.2-alpine3.17
    container_name: awari-postgres
    ports:
      - "5431:5432"
    healthcheck:
      test: ["CMD", "pg_isready", "-U", "postgres"]
      interval: 5s
      retries: 5
    env_file:
      - data/postgresql/.env
    volumes:
      - ./data/postgresql/data:/var/lib/postgresql/data
      - ./data/postgresql/create-multiple-postgresql-databases.sh:/docker-entrypoint-initdb.d/01-create-db.sh
    networks:
      awari-pipelines:
        ipv4_address: 172.18.0.6
  
  # Jupyter Lab Service
  awari-jupyterlab:
    build:
      dockerfile: data/jupyterlab/Dockerfile
    volumes:
     - ./jupyter-notebooks/:/notebooks
    ports:
      - "8888:8888"
      - "20020:20020"
    environment:
      - JUPYTER_ENABLE_LAB=1
      - SPARK_MASTER_IP=127.0.0.1
    command: start-notebook.sh --NotebookApp.notebook_dir=/notebooks --NotebookApp.token='' --NotebookApp.password=''
    networks:
      awari-pipelines:
        ipv4_address: 172.18.0.10

  # Kafkta Services
  awari-zookeeper:
    image: wurstmeister/zookeeper:3.4.6
    ports:
     - "2181:2181"
    networks:
      awari-pipelines:
        ipv4_address: 172.18.0.22

  awari-kafka:
    image: wurstmeister/kafka:2.13-2.8.1
    ports:
     - "9092:9092"
    expose:
     - "9093"
    environment:
      KAFKA_ADVERTISED_LISTENERS: INSIDE://awari-kafka:9093,OUTSIDE://localhost:9092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
      KAFKA_LISTENERS: INSIDE://0.0.0.0:9093,OUTSIDE://0.0.0.0:9092
      KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
      KAFKA_ZOOKEEPER_CONNECT: awari-zookeeper:2181
    networks:
      awari-pipelines:
        ipv4_address: 172.18.0.23

  awari-kafka-ui:
    image: provectuslabs/kafka-ui:v0.5.0
    container_name: awari-kafka-ui
    ports:
      - "9099:8080"
    environment:
      - KAFKA_CLUSTERS_0_NAME=local
      - KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS=awari-kafka:9093
    depends_on:
      - awari-kafka
    networks:
      awari-pipelines:
        ipv4_address: 172.18.0.24

  #Metabase Service
  metabase-app:
    image: metabase/metabase:v0.45.3
    restart: always
    ports:
      - 3001:3000
    volumes:
      # declare your mount volume /host/dir:/container/dir
      - ./data/metabase/data:/metabase-data
    env_file:
      - ./data/metabase/.env
    depends_on:
      - awari-postgres
    links:
      - awari-postgres
    networks:
      awari-pipelines:
        ipv4_address: 172.18.0.25