Avoid modifying log level globally (#1944)

* Avoid modifying log level globally

* Address get_job_logs

* Fix integration tests

Author: droctothorpe

sdk/python/kubeflow/training/api/training_client.py

@@ -24,8 +24,7 @@
 from kubeflow.training.constants import constants
 from kubeflow.training.utils import utils
 
-logging.basicConfig(format="%(message)s")
-logging.getLogger().setLevel(logging.INFO)
+logger = logging.getLogger(__name__)
 
 status_logger = utils.StatusLogger(
     header="{:<30.30} {:<20.20} {}".format("NAME", "STATE", "TIME"),
@@ -222,7 +221,7 @@ def create_job(
                 f"Failed to create {job_kind}: {namespace}/{job.metadata.name}"
             )
 
-        logging.info(f"{job_kind} {namespace}/{job.metadata.name} has been created")
+        logger.debug(f"{job_kind} {namespace}/{job.metadata.name} has been created")
 
     def get_job(
         self,
@@ -771,7 +770,7 @@ def get_job_logs(
         replica_index: Optional[int] = None,
         follow: bool = False,
         timeout: int = constants.DEFAULT_TIMEOUT,
-    ):
+    ) -> Dict[str, str]:
         """Print the training logs for the Job. By default it returns logs from
         the `master` pod.
 
@@ -801,6 +800,10 @@ def get_job_logs(
             timeout: Optional, Kubernetes API server timeout in seconds
                 to execute the request.
 
+        Returns:
+            Dict[str, str]: A dictionary in which the keys are pod names and the
+            values are the corresponding logs.
+
         Raises:
             ValueError: Job replica type is invalid.
             TimeoutError: Timeout to get Job pods.
@@ -819,6 +822,7 @@ def get_job_logs(
             timeout=timeout,
         )
 
+        logs_dict = {}
         if pods and follow:
             log_streams = []
             for pod in pods:
@@ -849,7 +853,7 @@ def get_job_logs(
                             if logline is None:
                                 finished[index] = True
                                 break
-                            logging.info("[Pod %s]: %s", pods[index], logline)
+                            print(f"[Pod {pods[index]}]: {logline}")
                         except queue.Empty:
                             break
         elif pods:
@@ -860,10 +864,12 @@ def get_job_logs(
                         namespace,
                         container=constants.JOB_PARAMETERS[job_kind]["container"],
                     )
-                    logging.info("The logs of pod %s:\n %s", pod, pod_logs)
+                    logs_dict[pod] = pod_logs
                 except Exception:
                     raise RuntimeError(f"Failed to read logs for pod {namespace}/{pod}")
 
+        return logs_dict
+
     def update_job(
         self,
         job: constants.JOB_MODELS_TYPE,
@@ -908,7 +914,7 @@ def update_job(
         except Exception:
             raise RuntimeError(f"Failed to update {job_kind}: {namespace}/{name}")
 
-        logging.info(f"{job_kind} {namespace}/{name} has been updated")
+        logger.debug(f"{job_kind} {namespace}/{name} has been updated")
 
     def delete_job(
         self,
@@ -950,4 +956,4 @@ def delete_job(
         except Exception:
             raise RuntimeError(f"Failed to delete {job_kind}: {namespace}/{name}")
 
-        logging.info(f"{job_kind} {namespace}/{name} has been deleted")
+        logger.debug(f"{job_kind} {namespace}/{name} has been deleted")

sdk/python/kubeflow/training/utils/utils.py

@@ -25,8 +25,7 @@
 from kubeflow.training import models
 
 
-logging.basicConfig(format="%(message)s")
-logging.getLogger().setLevel(logging.INFO)
+logger = logging.getLogger(__name__)
 
 
 class StatusLogger:
@@ -39,9 +38,9 @@ def __init__(self, header, column_format):
 
     def __call__(self, *values):
         if self.first_call:
-            logging.info(self.header)
+            logger.debug(self.header)
             self.first_call = False
-        logging.info(self.column_format.format(*values))
+        logger.debug(self.column_format.format(*values))
 
 
 class FakeResponse: