Fix bugs with Redis in docker and Kamailio startup and configuration, add WebUI to autostart

build/docker/Dockerfile

@@ -1,14 +1,12 @@
-FROM debian:bullseye
+FROM debian:bullseye AS build
 LABEL maintainer="Minh Minh <[email protected]>"
-ENV LIBRE_CONTAINERIZED 1
-ENV LIBRE_BUILTIN_FIREWALL 0
-ENV LIBRE_REDIS 1
+
+# ==================== BUILD ====================
 
 # BASE SOFTWARE
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -yq install \
 # base
-    git curl wget lsof vim redis procps\
-    sngrep tcpdump net-tools rsyslog logrotate rsync nftables chrony \
+    git curl wget \
 # build
     build-essential make cmake gnupg2 automake autoconf g++ gcc 'libtool-bin|libtool' pkg-config \
 # general
@@ -26,12 +24,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -yq install \
 # kams
     flex gdb libxml2-dev libunistring-dev libhiredis-dev
 
-RUN mkdir -p /run/redis /opt/libresbc /var/log/libresbc/cdr
-COPY callng /opt/libresbc/callng
-COPY liberator /opt/libresbc/liberator
-COPY build/ansible/roles/platform/files/modules.conf /opt/libresbc/modules.conf
-
-# FREESWITCH
+# Download FreeSWITCH & add modules
 RUN git clone https://github.com/signalwire/libks /usr/src/libs/libks && \
     git clone --branch v1.13.17 https://github.com/freeswitch/sofia-sip.git /usr/src/libs/sofia-sip && \
     git clone https://github.com/freeswitch/spandsp /usr/src/libs/spandsp && \
@@ -42,33 +35,82 @@ RUN git clone https://github.com/signalwire/libks /usr/src/libs/libks && \
     cp /usr/include/opencore-amrnb/interf_enc.h /usr/src/freeswitch/src/mod/codecs/mod_amr/interf_enc.h && \
     cp /usr/include/opencore-amrnb/interf_dec.h /usr/src/freeswitch/src/mod/codecs/mod_amr/interf_dec.h
 
-RUN cd /usr/src/libs/libks && cmake . -DCMAKE_INSTALL_PREFIX=/usr -DWITH_LIBBACKTRACE=1 && make install && \
-    cd /usr/src/libs/sofia-sip && ./bootstrap.sh && ./configure CFLAGS="-g -ggdb" --with-pic --with-glib=no --without-doxygen --disable-stun --prefix=/usr && make -j`nproc --all` && make install && \
-    cd /usr/src/libs/spandsp && git checkout 0d2e6ac && ./bootstrap.sh && ./configure CFLAGS="-g -ggdb" --with-pic --prefix=/usr && make -j`nproc --all` && make install && \
-    cd /usr/src/libs/signalwire-c && PKG_CONFIG_PATH=/usr/lib/pkgconfig cmake . -DCMAKE_INSTALL_PREFIX=/usr && make install && \
-    cd /usr/src/freeswitch && cp /opt/libresbc/modules.conf /usr/src/freeswitch/modules.conf && \
-    ./bootstrap.sh -j && ./configure -C --prefix=/usr/local --with-rundir=/run/freeswitch --with-logfiledir=/var/log/freeswitch/ --enable-64 --with-openssl && make -j`nproc` && make install
+# Build FreeSWITCH
+RUN cd /usr/src/libs/libks && cmake . -DCMAKE_INSTALL_PREFIX=/usr -DWITH_LIBBACKTRACE=1 && make install
+RUN cd /usr/src/libs/sofia-sip && ./bootstrap.sh && ./configure CFLAGS="-g -ggdb" --with-pic --with-glib=no --without-doxygen --disable-stun --prefix=/usr && make -j`nproc --all` && make install
+RUN cd /usr/src/libs/spandsp && git checkout 0d2e6ac && ./bootstrap.sh && ./configure CFLAGS="-g -ggdb" --with-pic --prefix=/usr && make -j`nproc --all` && make install
+RUN cd /usr/src/libs/signalwire-c && PKG_CONFIG_PATH=/usr/lib/pkgconfig cmake . -DCMAKE_INSTALL_PREFIX=/usr && make install
+RUN --mount=type=bind,source=build/ansible/roles/platform/files/modules.conf,target=/usr/src/freeswitch/modules.conf \
+    cd /usr/src/freeswitch && ./bootstrap.sh -j && ./configure -C --prefix=/usr/local --with-rundir=/run/freeswitch --with-logfiledir=/var/log/freeswitch/ --enable-64 --with-openssl && make -j`nproc` && make install
 
+# Download and build G729 codec module
 RUN git clone https://github.com/hnimminh/mod_bcg729.git /usr/local/src/mod_bcg729 && cd /usr/local/src/mod_bcg729 && make && make install
 
-# KAMAILIO
+# Download and build Kamailio
 RUN curl https://www.kamailio.org/pub/kamailio/5.7.1/src/kamailio-5.7.1_src.tar.gz -o /usr/local/src/kamailio-5.7.1_src.tar.gz && \
-    tar -xzvf /usr/local/src/kamailio-5.7.1_src.tar.gz -C /usr/local/src && cd /usr/local/src/kamailio-5.7.1 && \
+    tar -xzvf /usr/local/src/kamailio-5.7.1_src.tar.gz -C /usr/local/src
+RUN cd /usr/local/src/kamailio-5.7.1 && \
     make cfg && make include_modules="jsonrpcs ctl kex corex tm tmx outbound sl rr pv maxfwd topoh dialog usrloc registrar textops textopsx siputils sanity uac kemix auth nathelper tls debugger htable pike xlog app_lua regex utils" cfg && \
     make all && make install
 
-RUN chmod +x /opt/libresbc/callng/requirement.sh && /opt/libresbc/callng/requirement.sh &&\
+# Install LUA & Python requirements
+RUN --mount=type=bind,rw,source=callng/requirement.sh,target=/opt/libresbc/callng/requirement.sh \
+    chmod +x /opt/libresbc/callng/requirement.sh && /opt/libresbc/callng/requirement.sh
+RUN --mount=type=bind,source=liberator/requirements.txt,target=/opt/libresbc/liberator/requirements.txt \
     pip3 install -r /opt/libresbc/liberator/requirements.txt
 
-# LAYOUT & CLEANUP
-RUN rm -rf /usr/src/freeswitch-1.10.9.tar.gz /usr/local/freeswitch/conf/* /usr/local/src/kamailio* && \
-    apt-get clean && rm -rf /var/lib/apt/lists/* && \
+# Install Go
+RUN curl -L https://go.dev/dl/go1.23.2.linux-amd64.tar.gz -o /usr/local/go1.23.2.linux-amd64.tar.gz && tar -xzf /usr/local/go1.23.2.linux-amd64.tar.gz -C /usr/local
+
+# Build LibreSBC WebUI
+COPY ./webui /opt/libresbc/webui
+RUN cd /opt/libresbc/webui && /usr/local/go/bin/go build -o /opt/libresbc/webui/webuisrv
+
+
+# ==================== RUNTIME ====================
+
+FROM debian:bullseye-slim
+LABEL maintainer="Minh Minh <[email protected]>"
+ENV LIBRE_CONTAINERIZED 1
+ENV LIBRE_BUILTIN_FIREWALL 0
+ENV LIBRE_REDIS 1
+
+# Install runtime packages
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -yq --no-install-recommends --no-install-suggests install \
+    lsof vim redis procps sngrep tcpdump net-tools iproute2 curl \
+    python3 lua5.2 && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Copy files from build stage
+COPY --from=build /usr/lib/x86_64-linux-gnu	     /usr/lib/x86_64-linux-gnu
+COPY --from=build /usr/lib/lib*			        /usr/lib/
+COPY --from=build /usr/local/bin		        /usr/local/bin
+COPY --from=build /usr/local/lib/freeswitch	    /usr/local/lib/freeswitch
+COPY --from=build /usr/local/var/lib/freeswitch	/usr/local/var/lib/freeswitch
+COPY --from=build /usr/local/share/freeswitch	/usr/local/share/freeswitch
+COPY --from=build /usr/local/etc/freeswitch 	/usr/local/etc/freeswitch
+COPY --from=build /usr/local/lib/lib*		    /usr/local/lib/
+COPY --from=build /usr/local/sbin/kam*		    /usr/local/sbin/
+COPY --from=build /usr/local/share/kamailio	    /usr/local/share/kamailio
+COPY --from=build /usr/local/etc/kamailio	    /usr/local/etc/kamailio
+COPY --from=build /usr/local/lib64/kamailio	    /usr/local/lib64/kamailio
+COPY --from=build /usr/local/lib/lua		    /usr/local/lib/lua
+COPY --from=build /usr/local/share/lua		    /usr/local/share/lua
+COPY --from=build /usr/local/lib/python3.9	    /usr/local/lib/python3.9
+COPY --from=build /opt/libresbc/webui		    /opt/libresbc/webui
+
+COPY ./callng		/opt/libresbc/callng
+COPY ./liberator	/opt/libresbc/liberator
+
+# LAYOUT
+RUN mkdir -p /run/redis /var/log/libresbc/cdr && \
     ln -nfs /opt/libresbc/callng /usr/local/share/lua/5.2/callng && \
     ln -nfs /opt/libresbc/callng /usr/local/share/freeswitch/scripts/callng
 
+VOLUME ["/var/redis", "/var/tls"]
 WORKDIR /opt/libresbc/liberator
 CMD ["/usr/bin/python3", "/opt/libresbc/liberator/main.py"]
 
 # docker build --platform linux/amd64 -t hnimminh/libresbc:latest -f build/docker/Dockerfile .
 # docker tag hnimminh/libresbc:latest hnimminh/libresbc:0.7.1.c
-# docker run --env-file ../libre.env --cap-add NET_ADMIN --cap-add SYS_NICE --network host --name libresbc hnimminh/libresbc:latest
+# docker run --env-file build/docker/libre.env --cap-add NET_ADMIN --cap-add SYS_NICE --network host --name libresbc -volume libresbc:/var/redis --volume libresbc:/var/tls hnimminh/libresbc:latest

build/docker/docker-compose.yml

@@ -14,8 +14,8 @@ services:
     restart: always
     image: hnimminh/libresbc:latest
     volumes:
-      - ./liberator:/opt/libresbc/liberator
-      - ./callng:/opt/libresbc/callng
+      - libresbc:/var/redis
+      - libresbc:/var/tls
     network_mode: host
     cap_add:
       - NET_ADMIN

build/docker/libre.env

@@ -1,8 +1,7 @@
 REDIS_HOST=127.0.0.1
 REDIS_PORT=6379
 REDIS_DB=0
-SCAN_COUNT=1000
-REDIS_TIMEOUT=5
 NODEID=devsbc
 LOGSTACKS=CONSOLE
-LIBRE_REDIS=NO
+LIBRE_REDIS=YES
+LIBRE_WEBUI=YES

liberator/basemgr.py

@@ -18,9 +18,9 @@
 from jinja2 import Environment, FileSystemLoader
 from ipaddress import ip_address as IPvAddress, ip_network as IPvNetwork
 from configuration import (NODEID, CHANGE_CFG_CHANNEL, NODEID_CHANNEL, SECURITY_CHANNEL, ESL_HOST, ESL_PORT,
-    REDIS_HOST, REDIS_PORT, REDIS_DB, REDIS_PASSWORD, REDIS_TIMEOUT, LOGLEVEL, LOGSTACKS,
-    CONTAINERIZED, BUILTIN_FIREWALL, LIBRE_REDIS,
-)
+                           REDIS_HOST, REDIS_PORT, REDIS_DB, REDIS_PASSWORD, REDIS_TIMEOUT, LOGLEVEL, LOGSTACKS,
+                           CONTAINERIZED, BUILTIN_FIREWALL, LIBRE_REDIS, LIBRE_WEBUI,
+                           )
 from utilities import logger, threaded, listify, fieldjsonify, stringify, bdecode, jsonhash, randomstr
 
 
@@ -243,7 +243,7 @@ def fsinstance(data):
         fsrun = Popen(['/usr/local/bin/freeswitch', '-nc', '-reincarnate'], stdout=PIPE, stderr=PIPE)
         _, stderr = bdecode(fsrun.communicate())
 
-        if stderr and not stderr.endswith('Backgrounding.'):
+        if stderr and not stderr.endswith('Backgrounding.\n'):
             result = False
             stderr = stderr.replace('\n', '')
             logger.error(f"module=liberator, space=basemgr, action=fsinstance.fsrun, error={stderr}")
@@ -315,6 +315,7 @@ def kaminstance(data):
             _pidfile = f'{PIDDIR}/{_layer}.pid'
             _cfgfile = f'{CFGDIR}/{_layer}.cfg'
             _luafile = f'{CFGDIR}/{_layer}.lua'
+            _tlsfile = f'{CFGDIR}/{layer}.tls.cfg'
 
             kamend = Popen([pidkill, '-F', _pidfile], stdout=PIPE, stderr=PIPE)
             _, stderr = bdecode(kamend.communicate())
@@ -325,7 +326,9 @@ def kaminstance(data):
 
             cfgdel = osdelete(_cfgfile)
             luadel = osdelete(_luafile)
-            logger.info(f"module=liberator, space=basemgr, action=kaminstance.filedel, requestid={requestid}, cfgdel={cfgdel}, luadel={luadel}")
+            tlsdel = osdelete(_tlsfile)
+            piddel = osdelete(_pidfile)
+            logger.info(f"module=liberator, space=basemgr, action=kaminstance.filedel, requestid={requestid}, cfgdel={cfgdel}, luadel={luadel}, tlsdel={tlsdel}, piddel={piddel}")
         # ------------------------------------------------------------
         # LAUNCH THE NEW INSTANCE
         # ------------------------------------------------------------
@@ -335,6 +338,7 @@ def kaminstance(data):
             pidfile = f'{PIDDIR}/{layer}.pid'
             cfgfile = f'{CFGDIR}/{layer}.cfg'
             luafile = f'{CFGDIR}/{layer}.lua'
+            tlsfile = f'{CFGDIR}/{layer}.tls.cfg'
 
             kamcfgs = jsonhash(rdbconn.hgetall(f'access:service:{layer}'))
             netaliases = fieldjsonify(rdbconn.hget(f'base:netalias:{kamcfgs.get("sip_address")}', 'addresses'))
@@ -368,6 +372,11 @@ def kaminstance(data):
             luatemplate = _KAM.get_template("layer.j2.lua")
             luastream = luatemplate.render(_KAMCONST=_KAMCONST, kamcfgs=kamcfgs, layer=layer, swipaddrs=swipaddrs, jsonpolicies=json.dumps(policies), dftdomain=dftdomain)
             with open(luafile, 'w') as lf: lf.write(luastream)
+            # TLS configuration
+            if 'tls' in kamcfgs.get('transports'):
+                tlstemplate = _KAM.get_template("layer.j2.tls.cfg")
+                tlsstream = tlstemplate.render(_KAMCONST=_KAMCONST, kamcfgs=kamcfgs, layer=layer)
+                with open(tlsfile, 'w') as tf: tf.write(tlsstream)
 
             kamrun = Popen([kambin, '-S', '-M', '16', '-P', pidfile, '-f', cfgfile], stdout=PIPE, stderr=PIPE)
             _, stderr = bdecode(kamrun.communicate())
@@ -395,7 +404,7 @@ def rdbinstance():
     try:
         logger.info(f"module=liberator, space=basemgr, node={NODEID}, action=rdbinstance, state=initiating")
         rdbrun = Popen(['/usr/bin/redis-server', '--bind', '127.0.0.1', '--port', '6379', '--pidfile', '/run/redis/redis.pid', '--unixsocket',
-                        '/run/redis/redis.sock', '--unixsocketperm', '755', '--dbfilename', 'libresbc.rdb', '--dir', '/run/redis', '--loglevel', 'warning'])
+                        '/run/redis/redis.sock', '--unixsocketperm', '755', '--appendfilename', 'libresbc.aof', '--dir', '/var/redis',  '--appendonly', 'yes', '--loglevel', 'warning'])
         _, stderr = bdecode(rdbrun.communicate())
         if stderr:
             logger.error(f"module=liberator, space=basemgr, action=rdbinstance.rdbrun, error={stderr}")
@@ -404,27 +413,57 @@ def rdbinstance():
     except Exception as e:
         logger.critical(f'module=liberator, space=basemgr, action=exception, exception={e}, tracings={traceback.format_exc()}')
 
+#-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+# WEB USER INTERFACE
+#-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
+
+@threaded
+def webui():
+    if not LIBRE_WEBUI:
+        logger.info(f"module=liberator, space=basemgr, action=webui, message=[skip action since Web UI is disabled]")
+        return
+
+    try:
+        logger.info(f"module=liberator, space=basemgr, node={NODEID}, action=webui, state=initiating")
+        webuirun = Popen(['/opt/libresbc/webui/webuisrv', '-libresbc', 'http://127.0.0.1:8080'])
+        _, stderr = bdecode(webuirun.communicate())
+        if stderr:
+            logger.error(f"module=liberator, space=basemgr, action=webui, error={stderr}")
+        else:
+            logger.info(f"module=liberator, space=basemgr, action=webui, result=success")
+    except Exception as e:
+        logger.critical(f'module=liberator, space=basemgr, action=exception, exception={e}, tracings={traceback.format_exc()}')
 
 #-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
 # BASE RESOURCE STARTUP
 #-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
 _NGLUA = Environment(loader=FileSystemLoader('nglua'))
+_REDIS_TIMEOUT = 50 #seconds
 @threaded
 def basestartup():
     result = False
     try:
         logger.info(f"module=liberator, space=basemgr, node={NODEID}, action=basestartup, state=initiating")
         data = {'portion': 'liberator:startup', 'requestid': '00000000-0000-0000-0000-000000000000'}
         rdbinstance()
+        for t in range(1, _REDIS_TIMEOUT//5):
+            try:
+                if rdbconn.ping():
+                    break
+            except redis.ConnectionError:
+                logger.info(f"module=liberator, space=basemgr, action=rdbinstance, result=Waiting for Redis (attempt {t})...")
+                time.sleep(5)
+        if not rdbconn.ping():
+            logger.error(f'module=liberator, space=basemgr, action=exception, result="Redis has not started in {_REDIS_TIMEOUT} seconds. Other modules can not be loaded."')
+            return
+        webui()
         fsinstance(data)
         nftupdate(data)
         layers = rdbconn.smembers('nameset:access:service')
         for layer in layers:
             data.update({'layer': layer, '_layer': layer})
             kaminstance(data)
         result = True
-    except redis.RedisError as e:
-        time.sleep(10)
     except Exception as e:
         logger.critical(f'module=liberator, space=basemgr, action=exception, exception={e}, tracings={traceback.format_exc()}')
         time.sleep(5)

liberator/configuration.py

@@ -48,6 +48,11 @@
 if _LIBRE_REDIS and _LIBRE_REDIS.upper() in ['TRUE', '1', 'YES']:
     LIBRE_REDIS = True
 
+_LIBRE_WEBUI = os.getenv('LIBRE_WEBUI')
+LIBRE_WEBUI = False
+if _LIBRE_WEBUI and _LIBRE_WEBUI.upper() in ['TRUE', '1', 'YES']:
+    LIBRE_WEBUI = True
+
 _BUILTIN_FIREWALL = os.getenv('LIBRE_BUILTIN_FIREWALL')
 BUILTIN_FIREWALL = True
 if _BUILTIN_FIREWALL and _BUILTIN_FIREWALL.upper() in ['FALSE', '0', 'NO']:

liberator/kamcfg/layer.j2.tls.cfg

@@ -0,0 +1,16 @@
+[server:default]
+method = {{kamcfgs.tls.method}}
+verify_certificate = no
+require_certificate = no
+private_key = {{kamcfgs.tls.key}}
+certificate = {{kamcfgs.tls.cert}}
+{%- if kamcfgs.tlssni %}
+server_name = {{kamcfgs.tls.sni}}
+{%- endif %}
+#ca_list = /var/tls/cacert.pem
+#crl = /var/tls/crl.pem
+
+[client:default]
+method = {{kamcfgs.tls.method}}
+verify_certificate = no
+require_certificate = no