diff --git a/run_marqo.sh b/run_marqo.sh index 5b393a4bc..01d3adbe2 100644 --- a/run_marqo.sh +++ b/run_marqo.sh @@ -1,28 +1,44 @@ #!/bin/bash -#source /opt/bash-utils/logger.sh + +# set the default value to info and convert to lower case +export MARQO_LOG_LEVEL=${MARQO_LOG_LEVEL:-info} +MARQO_LOG_LEVEL=`echo "$MARQO_LOG_LEVEL" | tr '[:upper:]' '[:lower:]'` + export PYTHONPATH="${PYTHONPATH}:/app/src/" export CUDA_HOME=/usr/local/cuda/ export LD_LIBRARY_PATH=${CUDA_HOME}/lib64 export PATH=${CUDA_HOME}/bin:${PATH} trap "bash /app/scripts/shutdown.sh; exit" SIGTERM SIGINT -echo "Python packages:" -pip freeze + +if [ "$MARQO_LOG_LEVEL" = "debug" ]; then + echo "Python packages:" + pip freeze +fi function wait_for_process () { local max_retries=30 local n_restarts_before_sigkill=3 local process_name="$1" local retries=0 - while ! [[ $(docker ps -a | grep CONTAINER) ]] >/dev/null && ((retries < max_retries)); do - echo "Process $process_name is not running yet. Retrying in 1 seconds" - echo "Retry $retries of a maximum of $max_retries retries" + while ! [[ $(docker ps -a | grep CONTAINER) ]] > /dev/null 2>&1 && ((retries < max_retries)); do + if [ "$MARQO_LOG_LEVEL" = "debug" ]; then + echo "Process $process_name is not running yet. Retrying in 1 seconds" + echo "Retry $retries of a maximum of $max_retries retries" + fi + echo "Preparing to start Marqo-OS..." ((retries=retries+1)) if ((retries >= n_restarts_before_sigkill)); then - echo "sending SIGKILL to dockerd and restarting " - ps axf | grep docker | grep -v grep | awk '{print "kill -9 " $1}' | sh; rm /var/run/docker.pid; dockerd & + if [ "$MARQO_LOG_LEVEL" = "debug" ]; then + echo "sending SIGKILL to dockerd and restarting " + fi + ps axf | grep docker | grep -v grep | awk '{print "kill -9 " $1}' | sh; rm /var/run/docker.pid; dockerd > /dev/null 2>&1 & else - dockerd & + if [ "$MARQO_LOG_LEVEL" = "debug" ]; then + dockerd & + else + dockerd > /dev/null 2>&1 & + fi fi sleep 3 if ((retries >= max_retries)); then @@ -45,13 +61,18 @@ if [[ ! $OPENSEARCH_URL ]]; then wait "$setup_dind_pid" fi - echo "Starting supervisor" + if [ "$MARQO_LOG_LEVEL" = "debug" ]; then + echo "Starting supervisor" + fi + /usr/bin/supervisord -n >> /dev/null 2>&1 & - dockerd & - echo "called dockerd command" + dockerd > /dev/null 2>&1 & + + if [ "$MARQO_LOG_LEVEL" = "debug" ]; then + echo "Called dockerd command. Waiting for dockerd to start." + fi - echo "Waiting for processes to be running" processes=(dockerd) for process in "${processes[@]}"; do wait_for_process "$process" @@ -59,28 +80,39 @@ if [[ ! $OPENSEARCH_URL ]]; then echo "$process is not running after max time" exit 1 else - echo "$process is running" + if [ "$MARQO_LOG_LEVEL" = "debug" ]; then + echo "$process is running" + fi fi done OPENSEARCH_URL="https://localhost:9200" OPENSEARCH_IS_INTERNAL=True if [[ $(docker ps -a | grep marqo-os) ]]; then if [[ $(docker ps -a | grep marqo-os | grep -v Up) ]]; then - docker start marqo-os & + docker start marqo-os > /dev/null 2>&1 & until [[ $(curl -v --silent --insecure $OPENSEARCH_URL 2>&1 | grep Unauthorized) ]]; do sleep 0.1; done; - echo "Opensearch started" + echo "Marqo-OS started" fi - echo "OpenSearch is running" + echo "Marqo-OS is running" else - echo "OpenSearch not found; running OpenSearch" - docker run --name marqo-os -id -p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" marqoai/marqo-os:0.0.3 & - docker start marqo-os & + echo "Marqo-OS not found; starting Marqo-OS..." + if [ "$MARQO_LOG_LEVEL" = "debug" ]; then + docker run --name marqo-os -id -p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" marqoai/marqo-os:0.0.3 & + else + docker run --name marqo-os -id -p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" marqoai/marqo-os:0.0.3 > /dev/null 2>&1 & + fi + if [ "$MARQO_LOG_LEVEL" = "debug" ]; then + docker start marqo-os > /dev/null 2>&1 & + fi until [[ $(curl -v --silent --insecure $OPENSEARCH_URL 2>&1 | grep Unauthorized) ]]; do sleep 0.1; done; + echo "Marqo-OS started successfully." fi +else + echo "Found Marqo-OS URL. Skipping internal Marqo-OS configuration." fi export OPENSEARCH_URL @@ -88,9 +120,11 @@ export OPENSEARCH_IS_INTERNAL # Start up redis if [ "$MARQO_ENABLE_THROTTLING" != "FALSE" ]; then - echo "Starting redis-server" + echo "Starting Marqo throttling..." redis-server /etc/redis/redis.conf - echo "Called redis-server command" + if [ "$MARQO_LOG_LEVEL" = "debug" ]; then + echo "Called redis-server command" + fi start_time=$(($(date +%s%N)/1000000)) while true; do @@ -103,21 +137,22 @@ if [ "$MARQO_ENABLE_THROTTLING" != "FALSE" ]; then elapsed_time=$(expr $current_time - $start_time) if [ $elapsed_time -ge 2000 ]; then # Expected start time should be < 30ms in reality. - echo "redis-server failed to start within 2s. skipping." + # redis-server failed to start: + echo "Marqo throttling failed to start within 2s. Continuing without throttling." break fi sleep 0.1 done - echo "redis-server is now running" + # redis server is now running + echo "Marqo throttling successfully started." else - echo "Throttling has been disabled. Skipping redis-server start." + # skip starting Redis + echo "Marqo throttling has been disabled. Throttling start-up skipped." fi -# set the default value to info and convert to lower case -export MARQO_LOG_LEVEL=${MARQO_LOG_LEVEL:-info} -MARQO_LOG_LEVEL=`echo "$MARQO_LOG_LEVEL" | tr '[:upper:]' '[:lower:]'` + # Start the tensor search web app in the background cd /app/src/marqo/tensor_search || exit diff --git a/src/marqo/connections.py b/src/marqo/connections.py index c9561949a..b10cfdeb8 100644 --- a/src/marqo/connections.py +++ b/src/marqo/connections.py @@ -59,7 +59,8 @@ def init_from_app(self, host, port): self.faulty = False t1 = time.time() - logger.info(f"Took {((t1-t0)*1000):.3f}ms to connect to redis and load scripts.") + logger.debug(f"Took {((t1-t0)*1000):.3f}ms to connect to redis...") + logger.info(f"Took {((t1-t0)*1000):.3f}ms to connect to Marqo throttling and load scripts.") except Exception as e: logger.warn(generate_redis_warning(skipped_operation="loading throttling scripts", exc=e)) diff --git a/src/marqo/tensor_search/api.py b/src/marqo/tensor_search/api.py index ce016b016..27f874a11 100644 --- a/src/marqo/tensor_search/api.py +++ b/src/marqo/tensor_search/api.py @@ -1,8 +1,31 @@ """The API entrypoint for Tensor Search""" + +import warnings +from marqo.tensor_search.tensor_search_logging import get_logger +import logging + + +if get_logger(__name__).getEffectiveLevel() > logging.DEBUG: + # We need to suppress this warning before the dependency is imported + warnings.filterwarnings( + "ignore", + "Importing `GenerationMixin` from `src/transformers/generation_utils.py` " + "is deprecated and will be removed in Transformers v5. " + "Import as `from transformers import GenerationMixin` instead." + ) + warnings.filterwarnings( + "ignore", + ".*Unverified HTTPS request is being made to host 'localhost'.*" + ) + warnings.filterwarnings( + "ignore", + ".*Unverified HTTPS request is being made to host 'host.docker.internal'.*" + ) + import json import os import typing -from typing import List, Dict, Optional, Union +from typing import List, Dict, Optional import pydantic from fastapi import FastAPI, Query diff --git a/src/marqo/tensor_search/on_start_script.py b/src/marqo/tensor_search/on_start_script.py index 6cc75e752..9fe3360f4 100644 --- a/src/marqo/tensor_search/on_start_script.py +++ b/src/marqo/tensor_search/on_start_script.py @@ -8,7 +8,6 @@ from marqo.tensor_search import backend, index_meta_cache, utils from marqo import config from marqo.tensor_search.web import api_utils -from marqo._httprequests import HttpRequests from marqo import errors from marqo.tensor_search.throttling.redis_throttle import throttle from marqo.connections import redis_driver @@ -73,7 +72,7 @@ class CUDAAvailable: """checks the status of cuda """ - logger = get_logger('CUDA device summary') + logger = get_logger('DeviceSummary') def __init__(self): diff --git a/src/marqo/tensor_search/tensor_search_logging.py b/src/marqo/tensor_search/tensor_search_logging.py index 6527619e9..e36ad9662 100644 --- a/src/marqo/tensor_search/tensor_search_logging.py +++ b/src/marqo/tensor_search/tensor_search_logging.py @@ -1,6 +1,8 @@ import logging from marqo.tensor_search.utils import read_env_vars_and_defaults from marqo.errors import EnvVarError + + def get_logger(name): logging.basicConfig() logger = logging.getLogger(name) @@ -22,4 +24,4 @@ def get_logger(name): formatter = logging.Formatter( "{asctime} {threadName:>11} {levelname} {message}", style='{') - return logger \ No newline at end of file + return logger