diff --git a/src/azure-cli/azure/cli/command_modules/synapse/manual/_client_factory.py b/src/azure-cli/azure/cli/command_modules/synapse/manual/_client_factory.py index 2bbb2820db7..d6d09dc5353 100644 --- a/src/azure-cli/azure/cli/command_modules/synapse/manual/_client_factory.py +++ b/src/azure-cli/azure/cli/command_modules/synapse/manual/_client_factory.py @@ -105,8 +105,7 @@ def cf_kusto_pool(cli_ctx, *_): def synapse_spark_factory(cli_ctx, workspace_name, sparkpool_name): from azure.synapse.spark import SparkClient from azure.cli.core._profile import Profile - # Uncomment this line after SparkClient is fixed. See https://github.com/Azure/azure-cli/pull/30788 - # from azure.cli.core.auth.util import resource_to_scopes + from azure.cli.core.auth.util import resource_to_scopes from azure.cli.core.commands.client_factory import get_subscription_id subscription_id = get_subscription_id(cli_ctx) profile = Profile(cli_ctx=cli_ctx) @@ -115,8 +114,7 @@ def synapse_spark_factory(cli_ctx, workspace_name, sparkpool_name): credential=cred, endpoint='{}{}{}'.format("https://", workspace_name, cli_ctx.cloud.suffixes.synapse_analytics_endpoint), spark_pool_name=sparkpool_name, - # Uncomment this line after SparkClient is fixed - # credential_scopes=resource_to_scopes(cli_ctx.cloud.endpoints.synapse_analytics_resource_id) + credential_scopes=resource_to_scopes(cli_ctx.cloud.endpoints.synapse_analytics_resource_id) ) diff --git a/src/azure-cli/azure/cli/command_modules/synapse/tests/latest/recordings/test_spark_job.yaml b/src/azure-cli/azure/cli/command_modules/synapse/tests/latest/recordings/test_spark_job.yaml index a33f2c56b5d..f72fa8f6e36 100644 --- a/src/azure-cli/azure/cli/command_modules/synapse/tests/latest/recordings/test_spark_job.yaml +++ b/src/azure-cli/azure/cli/command_modules/synapse/tests/latest/recordings/test_spark_job.yaml @@ -1,11 +1,11 @@ interactions: - request: - body: '{"name": "WordCount_Java", "file": "abfss://aaa@foo.com/java/wordcount/wordcount.jar", - "className": "WordCount", "args": ["[abfss://aaa@foo.com/java/wordcount/shakespeare.txt,", - "abfss://aaa@foo.com/java/wordcount/result/]"], + body: '{"name": "WordCount_Java", "file": "abfss://filesystem01@ywtestaccount.dfs.core.windows.net/wordcount/wordcount_2.11-0.1.jar", + "className": "WordCount", "args": ["[abfss://filesystem01@ywtestaccount.dfs.core.windows.net/wordcount/input/shakespeare.txt,", + "abfss://filesystem01@ywtestaccount.dfs.core.windows.net/wordcount/output/]"], "conf": {"spark.dynamicAllocation.maxExecutors": "18"}, "driverMemory": "56g", "driverCores": 8, "executorMemory": "56g", "executorCores": 8, "numExecutors": - 2}' + 4}' headers: Accept: - application/json @@ -14,35 +14,35 @@ interactions: Connection: - keep-alive Content-Length: - - '530' + - '494' Content-Type: - application/json User-Agent: - - azsdk-python-synapse/0.2.0 Python/3.7.4 (Windows-10-10.0.19041-SP0) + - azsdk-python-synapse-spark/0.7.0 Python/3.10.11 (Windows-10-10.0.26100-SP0) method: POST - uri: https://testsynapseworkspace.dev.azuresynapse.net/livyApi/versions/2019-11-01-preview/sparkPools/testsparkpool/batches?detailed=true + uri: https://yanjuntestws001.dev.azuresynapse.net/livyApi/versions/2019-11-01-preview/sparkPools/sparkpooltest/batches?detailed=true response: body: - string: '{"livyInfo":{"notStartedAt":null,"startingAt":null,"runningAt":null,"deadAt":null,"successAt":null,"killedAt":null,"recoveringAt":null,"currentState":"not_started","jobCreationRequest":{"name":"WordCount_Java","file":"abfss://aaa@foo.com/java/wordcount/wordcount.jar","className":"WordCount","args":["[abfss://aaa@foo.com/java/wordcount/shakespeare.txt,","abfss://aaa@foo.com/java/wordcount/result/]"],"conf":{"spark.dynamicAllocation.maxExecutors":"18"},"driverMemory":"56g","driverCores":8,"executorMemory":"56g","executorCores":8,"numExecutors":2}},"name":"WordCount_Java","workspaceName":"testsynapseworkspace","computeName":"testsparkpool","sparkPoolName":"testsparkpool","submitterName":"aaa@foo.com","submitterId":"52379807-94ed-417a-9a37-b7751e3fb688","artifactId":"Livy","jobType":"SparkBatch","result":"Uncertain","schedulerInfo":{"submittedAt":"2021-06-07T09:13:40.7537681+00:00","scheduledAt":null,"endedAt":null,"cancellationRequestedAt":null,"currentState":"Queued"},"pluginInfo":{"preparationStartedAt":null,"resourceAcquisitionStartedAt":null,"submissionStartedAt":null,"monitoringStartedAt":null,"cleanupStartedAt":null,"currentState":"Preparation"},"tags":{},"id":423,"appId":null,"appInfo":null,"state":"not_started","log":null}' + string: '{"livyInfo":{"notStartedAt":null,"startingAt":null,"runningAt":null,"deadAt":null,"successAt":null,"killedAt":null,"recoveringAt":null,"currentState":"not_started","jobCreationRequest":{"name":"WordCount_Java","file":"abfss://filesystem01@ywtestaccount.dfs.core.windows.net/wordcount/wordcount_2.11-0.1.jar","className":"WordCount","args":["[abfss://filesystem01@ywtestaccount.dfs.core.windows.net/wordcount/input/shakespeare.txt,","abfss://filesystem01@ywtestaccount.dfs.core.windows.net/wordcount/output/]"],"conf":{"spark.dynamicAllocation.maxExecutors":"18"},"driverMemory":"56g","driverCores":8,"executorMemory":"56g","executorCores":8,"numExecutors":4}},"name":"WordCount_Java","workspaceName":"yanjuntestws001","computeName":"sparkpooltest","sparkPoolName":"sparkpooltest","submitterName":"v-weishuzhao@microsoft.com","submitterId":"6a33200f-2b27-4fda-8567-b0c70d532441","artifactId":"Livy","jobType":"SparkBatch","result":"Uncertain","schedulerInfo":{"submittedAt":"2025-02-24T09:47:41.6861206+00:00","queuedAt":"2025-02-24T09:47:42.1202441+00:00","scheduledAt":null,"endedAt":null,"cancellationRequestedAt":null,"currentState":"Queued"},"pluginInfo":{"preparationStartedAt":null,"resourceAcquisitionStartedAt":null,"submissionStartedAt":null,"monitoringStartedAt":null,"cleanupStartedAt":null,"currentState":"Preparation"},"tags":{},"id":0,"appId":null,"appInfo":null,"state":"not_started","log":null,"registeredSources":null}' headers: + content-length: + - '1435' content-type: - application/json; charset=utf-8 date: - - Mon, 07 Jun 2021 09:13:46 GMT + - Mon, 24 Feb 2025 09:47:41 GMT server: - Kestrel Microsoft-HTTPAPI/2.0 strict-transport-security: - max-age=31536000; includeSubDomains - transfer-encoding: - - chunked x-ms-activity-id: - - 9cca6699-d893-43de-b8ee-da7b09cb2ccd + - 6b3b0551-1c1b-4c3e-84a9-42160bb6ef24 x-ms-job-clusterrequested-on: - 1/1/0001 12:00:00 AM +00:00 x-ms-job-ended-on: - '' x-ms-job-internal-id: - - '423' + - '0' x-ms-job-livysubmission-on: - 1/1/0001 12:00:00 AM +00:00 x-ms-job-queued-on: @@ -54,13 +54,11 @@ interactions: x-ms-job-scheduler-state: - Queued x-ms-job-submitted-by-name: - - aaa@foo.com + - v-weishuzhao@microsoft.com x-ms-job-submitted-on: - - 6/7/2021 9:13:40 AM +00:00 + - 2/24/2025 9:47:41 AM +00:00 x-ms-job-type: - SparkServiceBatch - x-ms-response-time-ms: - - '5962' status: code: 200 message: OK @@ -74,31 +72,31 @@ interactions: Connection: - keep-alive User-Agent: - - azsdk-python-synapse/0.2.0 Python/3.7.4 (Windows-10-10.0.19041-SP0) + - azsdk-python-synapse-spark/0.7.0 Python/3.10.11 (Windows-10-10.0.26100-SP0) method: GET - uri: https://testsynapseworkspace.dev.azuresynapse.net/livyApi/versions/2019-11-01-preview/sparkPools/testsparkpool/batches/423?detailed=true + uri: https://yanjuntestws001.dev.azuresynapse.net/livyApi/versions/2019-11-01-preview/sparkPools/sparkpooltest/batches/0?detailed=true response: body: - string: '{"livyInfo":{"notStartedAt":null,"startingAt":null,"runningAt":null,"deadAt":null,"successAt":null,"killedAt":null,"recoveringAt":null,"currentState":"not_started","jobCreationRequest":{"name":"WordCount_Java","file":"abfss://aaa@foo.com/java/wordcount/wordcount.jar","className":"WordCount","args":["[abfss://aaa@foo.com/java/wordcount/shakespeare.txt,","abfss://aaa@foo.com/java/wordcount/result/]"],"conf":{"spark.dynamicAllocation.maxExecutors":"18"},"driverMemory":"56g","driverCores":8,"executorMemory":"56g","executorCores":8,"numExecutors":2}},"name":"WordCount_Java","workspaceName":"testsynapseworkspace","computeName":"testsparkpool","sparkPoolName":"testsparkpool","submitterName":"aaa@foo.com","submitterId":"52379807-94ed-417a-9a37-b7751e3fb688","artifactId":"Livy","jobType":"SparkBatch","result":"Uncertain","schedulerInfo":{"submittedAt":"2021-06-07T09:13:40.7537681+00:00","scheduledAt":null,"endedAt":null,"cancellationRequestedAt":null,"currentState":"Queued"},"pluginInfo":{"preparationStartedAt":null,"resourceAcquisitionStartedAt":null,"submissionStartedAt":null,"monitoringStartedAt":null,"cleanupStartedAt":null,"currentState":"Preparation"},"tags":{},"id":423,"appId":null,"appInfo":null,"state":"not_started","log":null}' + string: '{"livyInfo":{"notStartedAt":null,"startingAt":null,"runningAt":null,"deadAt":null,"successAt":null,"killedAt":null,"recoveringAt":null,"currentState":"not_started","jobCreationRequest":{"name":"WordCount_Java","file":"abfss://filesystem01@ywtestaccount.dfs.core.windows.net/wordcount/wordcount_2.11-0.1.jar","className":"WordCount","args":["[abfss://filesystem01@ywtestaccount.dfs.core.windows.net/wordcount/input/shakespeare.txt,","abfss://filesystem01@ywtestaccount.dfs.core.windows.net/wordcount/output/]"],"conf":{"spark.dynamicAllocation.maxExecutors":"18"},"driverMemory":"56g","driverCores":8,"executorMemory":"56g","executorCores":8,"numExecutors":4}},"name":"WordCount_Java","workspaceName":"yanjuntestws001","computeName":"sparkpooltest","sparkPoolName":"sparkpooltest","submitterName":"v-weishuzhao@microsoft.com","submitterId":"6a33200f-2b27-4fda-8567-b0c70d532441","artifactId":"Livy","jobType":"SparkBatch","result":"Uncertain","schedulerInfo":{"submittedAt":"2025-02-24T09:47:41.6861206+00:00","queuedAt":"2025-02-24T09:47:42.1202441+00:00","scheduledAt":null,"endedAt":null,"cancellationRequestedAt":null,"currentState":"Queued"},"pluginInfo":{"preparationStartedAt":null,"resourceAcquisitionStartedAt":null,"submissionStartedAt":null,"monitoringStartedAt":null,"cleanupStartedAt":null,"currentState":"Preparation"},"tags":{},"id":0,"appId":null,"appInfo":null,"state":"not_started","log":null,"registeredSources":null}' headers: + content-length: + - '1435' content-type: - application/json; charset=utf-8 date: - - Mon, 07 Jun 2021 09:13:47 GMT + - Mon, 24 Feb 2025 09:47:42 GMT server: - Kestrel Microsoft-HTTPAPI/2.0 strict-transport-security: - max-age=31536000; includeSubDomains - transfer-encoding: - - chunked x-ms-activity-id: - - bfe90a3c-3154-4224-9a15-0789957e6a59 + - e263039e-9304-44ad-9c4f-8f00322be2ca x-ms-job-clusterrequested-on: - 1/1/0001 12:00:00 AM +00:00 x-ms-job-ended-on: - '' x-ms-job-internal-id: - - '423' + - '0' x-ms-job-livysubmission-on: - 1/1/0001 12:00:00 AM +00:00 x-ms-job-queued-on: @@ -110,13 +108,11 @@ interactions: x-ms-job-scheduler-state: - Queued x-ms-job-submitted-by-name: - - aaa@foo.com + - v-weishuzhao@microsoft.com x-ms-job-submitted-on: - - 6/7/2021 9:13:40 AM +00:00 + - 2/24/2025 9:47:41 AM +00:00 x-ms-job-type: - SparkServiceBatch - x-ms-response-time-ms: - - '22' status: code: 200 message: OK @@ -130,187 +126,25 @@ interactions: Connection: - keep-alive User-Agent: - - azsdk-python-synapse/0.2.0 Python/3.7.4 (Windows-10-10.0.19041-SP0) + - azsdk-python-synapse-spark/0.7.0 Python/3.10.11 (Windows-10-10.0.26100-SP0) method: GET - uri: https://testsynapseworkspace.dev.azuresynapse.net/livyApi/versions/2019-11-01-preview/sparkPools/testsparkpool/batches?detailed=true + uri: https://yanjuntestws001.dev.azuresynapse.net/livyApi/versions/2019-11-01-preview/sparkPools/sparkpooltest/batches?detailed=true response: body: - string: '{"from":0,"total":20,"sessions":[{"livyInfo":{"notStartedAt":null,"startingAt":"2019-12-13T08:13:59.1387322+00:00","runningAt":"2019-12-13T08:14:43.2250458+00:00","deadAt":null,"successAt":"2019-12-13T08:15:16.3029723+00:00","killedAt":null,"recoveringAt":null,"currentState":"success","jobCreationRequest":{"name":"WordCount_Java","file":"abfss://aaa@foo.com/java/wordcount/wordcount.jar","className":"WordCount","args":["abfss://aaa@foo.com/java/wordcount/shakespeare.txt","abfss://aaa@foo.com/java/wordcount/result"],"conf":{},"driverMemory":"4G","driverCores":2,"executorMemory":"4G","executorCores":2,"numExecutors":2}},"name":"WordCount_Java","workspaceName":"testsynapseworkspace","computeName":"testsparkpool","sparkPoolName":"testsparkpool","submitterId":"ca728363-7297-4e35-a9c3-c6400837883e","artifactId":"Livy","jobType":"SparkBatch","result":"Succeeded","schedulerInfo":{"submittedAt":"2019-12-13T08:11:38.1206692+00:00","scheduledAt":"2019-12-13T08:11:39.2080482+00:00","endedAt":"2019-12-13T08:15:16.568297+00:00","cancellationRequestedAt":null,"currentState":"Ended"},"pluginInfo":{"preparationStartedAt":"2019-12-13T08:11:39.2230319+00:00","resourceAcquisitionStartedAt":"2019-12-13T08:11:39.4839299+00:00","submissionStartedAt":"2019-12-13T08:13:58.4588932+00:00","monitoringStartedAt":"2019-12-13T08:13:59.1558744+00:00","cleanupStartedAt":"2019-12-13T08:15:16.3435598+00:00","currentState":"Ended"},"tags":{},"id":0,"appId":"application_1576224831757_0002","appInfo":{"driverLogUrl":"http://53e52930712f4bd2a64da26879c4751c0076c436679:8042/node/containerlogs/container_1576224831757_0002_01_000001/trusted-service-user","sparkUiUrl":"http://53e52930712f4bd2a64da26879c4751c01223913297:8088/proxy/application_1576224831757_0002/"},"state":"success","log":["\t - tracking URL: http://53e52930712f4bd2a64da26879c4751c01223913297:8088/proxy/application_1576224831757_0002/","\t - user: trusted-service-user","19/12/13 08:14:22 INFO ShutdownHookManager: Shutdown - hook called","19/12/13 08:14:22 INFO ShutdownHookManager: Deleting directory - /tmp/spark-badf96dc-0837-41ba-9944-05f55277f611","19/12/13 08:14:22 INFO ShutdownHookManager: - Deleting directory /tmp/spark-04ab468f-e186-458c-91cd-ef6c97f5dff5","19/12/13 - 08:14:22 INFO MetricsSystemImpl: Stopping azure-file-system metrics system...","19/12/13 - 08:14:22 INFO MetricsSystemImpl: azure-file-system metrics system stopped.","19/12/13 - 08:14:22 INFO MetricsSystemImpl: azure-file-system metrics system shutdown - complete.","\nstderr: ","\nYARN Diagnostics: "]},{"livyInfo":{"notStartedAt":null,"startingAt":"2019-12-13T08:13:59.1372088+00:00","runningAt":"2019-12-13T08:14:54.2434247+00:00","deadAt":null,"successAt":"2019-12-13T08:15:38.4077027+00:00","killedAt":null,"recoveringAt":null,"currentState":"success","jobCreationRequest":{"name":"WordCount_DotNET","file":"abfss://aaa@foo.com/dotnet/wordcount/microsoft-spark-2.4.x-0.6.0.jar","className":"org.apache.spark.deploy.dotnet.DotnetRunner","args":["abfss://aaa@foo.com/dotnet/wordcount/wordcount.zip","WordCount","abfss://aaa@foo.com/dotnet/wordcount/shakespeare.txt","abfss://aaa@foo.com/dotnet/wordcount/result"],"conf":{},"driverMemory":"4G","driverCores":2,"executorMemory":"4G","executorCores":2,"numExecutors":2}},"name":"WordCount_DotNET","workspaceName":"testsynapseworkspace","computeName":"testsparkpool","sparkPoolName":"testsparkpool","submitterId":"ca728363-7297-4e35-a9c3-c6400837883e","artifactId":"Livy","jobType":"SparkBatch","result":"Succeeded","schedulerInfo":{"submittedAt":"2019-12-13T08:12:11.7282739+00:00","scheduledAt":"2019-12-13T08:12:12.8105461+00:00","endedAt":"2019-12-13T08:15:38.5885609+00:00","cancellationRequestedAt":null,"currentState":"Ended"},"pluginInfo":{"preparationStartedAt":"2019-12-13T08:12:12.8252672+00:00","resourceAcquisitionStartedAt":"2019-12-13T08:12:13.0474066+00:00","submissionStartedAt":"2019-12-13T08:13:58.738487+00:00","monitoringStartedAt":"2019-12-13T08:13:59.1818926+00:00","cleanupStartedAt":"2019-12-13T08:15:38.4532754+00:00","currentState":"Ended"},"tags":{},"id":1,"appId":"application_1576224831757_0001","appInfo":{"driverLogUrl":"http://53e52930712f4bd2a64da26879c4751c03817035090:8042/node/containerlogs/container_1576224831757_0001_01_000001/trusted-service-user","sparkUiUrl":"http://53e52930712f4bd2a64da26879c4751c01223913297:8088/proxy/application_1576224831757_0001/"},"state":"success","log":["\t - tracking URL: http://53e52930712f4bd2a64da26879c4751c01223913297:8088/proxy/application_1576224831757_0001/","\t - user: trusted-service-user","19/12/13 08:14:22 INFO ShutdownHookManager: Shutdown - hook called","19/12/13 08:14:22 INFO ShutdownHookManager: Deleting directory - /tmp/spark-080c8c63-d0dd-487d-a757-faf7b5efd9f1","19/12/13 08:14:22 INFO ShutdownHookManager: - Deleting directory /tmp/spark-aaed5be7-2685-4c28-a43c-8f37b9521d4f","19/12/13 - 08:14:22 INFO MetricsSystemImpl: Stopping azure-file-system metrics system...","19/12/13 - 08:14:22 INFO MetricsSystemImpl: azure-file-system metrics system stopped.","19/12/13 - 08:14:22 INFO MetricsSystemImpl: azure-file-system metrics system shutdown - complete.","\nstderr: ","\nYARN Diagnostics: "]},{"livyInfo":{"notStartedAt":null,"startingAt":"2019-12-25T03:56:21.0104385+00:00","runningAt":null,"deadAt":"2019-12-25T04:01:29.5722247+00:00","successAt":null,"killedAt":null,"recoveringAt":null,"currentState":"dead","jobCreationRequest":{"name":"WordCount_Java","file":"abfss://aaa@foo.com/java/wordcount/wordcount.jar","className":"WordCount","args":["abfss://aaa@foo.com/java/wordcount/shakespeare.txt","abfss://aaa@foo.com/java/wordcount/result/"],"conf":{},"driverMemory":"4g","driverCores":4,"executorMemory":"4g","executorCores":4,"numExecutors":2}},"name":"WordCount_Java","workspaceName":"testsynapseworkspace","computeName":"testsparkpool","sparkPoolName":"testsparkpool","submitterName":"aaa@foo.com","submitterId":"8f51e348-1fa6-4194-8d13-8e0368e3298a","artifactId":"Livy","jobType":"SparkBatch","result":"Failed","schedulerInfo":{"submittedAt":"2019-12-25T03:54:31.4237553+00:00","scheduledAt":"2019-12-25T03:54:32.5488235+00:00","endedAt":"2019-12-25T04:01:30.3397987+00:00","cancellationRequestedAt":null,"currentState":"Ended"},"pluginInfo":{"preparationStartedAt":"2019-12-25T03:54:32.5622451+00:00","resourceAcquisitionStartedAt":"2019-12-25T03:54:32.8795988+00:00","submissionStartedAt":"2019-12-25T03:56:20.3160953+00:00","monitoringStartedAt":"2019-12-25T03:56:21.0256995+00:00","cleanupStartedAt":"2019-12-25T04:01:29.592435+00:00","currentState":"Ended"},"errorInfo":[{"message":"[plugins.testsynapseworkspace.testsparkpool.3] - [Monitoring] Livy Endpoint=[http://171.1.155.33:8998/]. Livy Id=[0] Job failed - during run time with state=[dead].","errorCode":"LIVY_JOB_FAILED","source":"Dependency"}],"tags":{},"id":3,"appId":null,"appInfo":{"driverLogUrl":null,"sparkUiUrl":null},"state":"dead","log":["\tat - org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:933)","\tat org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","19/12/25 - 03:56:47 INFO ShutdownHookManager: Shutdown hook called","19/12/25 03:56:47 - INFO ShutdownHookManager: Deleting directory /tmp/spark-3ae6f965-dcbc-4b3e-9c59-d5c0da9d1d66","19/12/25 - 03:56:47 INFO MetricsSystemImpl: Stopping azure-file-system metrics system...","19/12/25 - 03:56:47 INFO MetricsSystemImpl: azure-file-system metrics system stopped.","19/12/25 - 03:56:47 INFO MetricsSystemImpl: azure-file-system metrics system shutdown - complete.","\nstderr: ","\nYARN Diagnostics: ","No YARN application is found - with tag livy-batch-0-7v8dgkxr in 300 seconds. This may be because 1) spark-submit - fail to submit application to YARN; or 2) YARN cluster doesn''t have enough - resources to start the application in time. Please check Livy log and YARN - log to know the details."]},{"livyInfo":{"notStartedAt":null,"startingAt":"2019-12-25T05:41:50.7810081+00:00","runningAt":"2019-12-25T05:42:56.9338323+00:00","deadAt":null,"successAt":"2019-12-25T05:43:41.0409981+00:00","killedAt":null,"recoveringAt":null,"currentState":"success","jobCreationRequest":{"name":"WordCount_Java","file":"abfss://aaa@foo.com/java/wordcount/wordcount.jar","className":"WordCount","args":["abfss://aaa@foo.com/java/wordcount/shakespeare.txt","abfss://aaa@foo.com/java/wordcount/result/"],"conf":{},"driverMemory":"4g","driverCores":4,"executorMemory":"4g","executorCores":4,"numExecutors":2}},"name":"WordCount_Java","workspaceName":"testsynapseworkspace","computeName":"testsparkpool","sparkPoolName":"testsparkpool","submitterName":"aaa@foo.com","submitterId":"8f51e348-1fa6-4194-8d13-8e0368e3298a","artifactId":"Livy","jobType":"SparkBatch","result":"Succeeded","schedulerInfo":{"submittedAt":"2019-12-25T05:39:13.7690381+00:00","scheduledAt":"2019-12-25T05:39:14.3364408+00:00","endedAt":"2019-12-25T05:43:41.7858451+00:00","cancellationRequestedAt":null,"currentState":"Ended"},"pluginInfo":{"preparationStartedAt":"2019-12-25T05:39:14.3510882+00:00","resourceAcquisitionStartedAt":"2019-12-25T05:39:14.5997804+00:00","submissionStartedAt":"2019-12-25T05:41:09.9672443+00:00","monitoringStartedAt":"2019-12-25T05:41:50.7975586+00:00","cleanupStartedAt":"2019-12-25T05:43:41.0567479+00:00","currentState":"Ended"},"tags":{},"id":4,"appId":"application_1577252484917_0001","appInfo":{"driverLogUrl":"http://59cc110db6dc46ac89103137e0c646e100149e16655:8042/node/containerlogs/container_1577252484917_0001_01_000001/trusted-service-user","sparkUiUrl":"http://59cc110db6dc46ac89103137e0c646e104164268157:8088/proxy/application_1577252484917_0001/"},"state":"success","log":["\t - tracking URL: http://59cc110db6dc46ac89103137e0c646e104164268157:8088/proxy/application_1577252484917_0001/","\t - user: trusted-service-user","19/12/25 05:42:14 INFO ShutdownHookManager: Shutdown - hook called","19/12/25 05:42:14 INFO ShutdownHookManager: Deleting directory - /tmp/spark-0c814ed9-49f2-4d82-8e8f-28b2f089f7fd","19/12/25 05:42:14 INFO ShutdownHookManager: - Deleting directory /tmp/spark-2f0a0fda-0d9d-40b2-a300-795d45a67025","19/12/25 - 05:42:14 INFO MetricsSystemImpl: Stopping azure-file-system metrics system...","19/12/25 - 05:42:14 INFO MetricsSystemImpl: azure-file-system metrics system stopped.","19/12/25 - 05:42:14 INFO MetricsSystemImpl: azure-file-system metrics system shutdown - complete.","\nstderr: ","\nYARN Diagnostics: "]},{"livyInfo":{"notStartedAt":null,"startingAt":"2019-12-25T05:48:37.8201633+00:00","runningAt":null,"deadAt":null,"successAt":null,"killedAt":null,"recoveringAt":null,"currentState":"killed","jobCreationRequest":{"name":"WordCount_Java","file":"abfss://aaa@foo.com/java/wordcount/wordcount.jar","className":"WordCount","args":["abfss://aaa@foo.com/java/wordcount/shakespeare.txt","abfss://aaa@foo.com/java/wordcount/result/"],"conf":{},"driverMemory":"4g","driverCores":4,"executorMemory":"4g","executorCores":4,"numExecutors":2}},"name":"WordCount_Java","workspaceName":"testsynapseworkspace","computeName":"testsparkpool","sparkPoolName":"testsparkpool","submitterName":"aaa@foo.com","submitterId":"8f51e348-1fa6-4194-8d13-8e0368e3298a","artifactId":"Livy","jobType":"SparkBatch","result":"Cancelled","schedulerInfo":{"submittedAt":"2019-12-25T05:48:34.6841508+00:00","scheduledAt":"2019-12-25T05:48:34.7956991+00:00","endedAt":"2019-12-25T05:48:48.664457+00:00","cancellationRequestedAt":"2019-12-25T05:48:47.8779744+00:00","currentState":"Ended"},"pluginInfo":{"preparationStartedAt":"2019-12-25T05:48:34.8085024+00:00","resourceAcquisitionStartedAt":"2019-12-25T05:48:35.0342873+00:00","submissionStartedAt":"2019-12-25T05:48:37.7430805+00:00","monitoringStartedAt":"2019-12-25T05:48:37.8617038+00:00","cleanupStartedAt":"2019-12-25T05:48:47.8779778+00:00","currentState":"Ended"},"tags":{},"id":5,"appId":null,"appInfo":{"driverLogUrl":null,"sparkUiUrl":null},"state":"killed","log":["stdout: - ","\nstderr: ","\nYARN Diagnostics: "]},{"livyInfo":{"notStartedAt":null,"startingAt":"2020-01-10T02:38:45.9207212+00:00","runningAt":"2020-01-10T02:40:03.1582266+00:00","deadAt":null,"successAt":"2020-01-10T02:41:20.3994418+00:00","killedAt":null,"recoveringAt":null,"currentState":"success","jobCreationRequest":{"name":"WordCount_Java","file":"abfss://aaa@foo.com/java/wordcount/wordcount.jar","className":"WordCount","args":["abfss://aaa@foo.com/java/wordcount/shakespeare.txt","abfss://aaa@foo.com/java/wordcount/result/"],"conf":{},"driverMemory":"4g","driverCores":4,"executorMemory":"4g","executorCores":4,"numExecutors":2}},"name":"WordCount_Java","workspaceName":"testsynapseworkspace","computeName":"testsparkpool","sparkPoolName":"testsparkpool","submitterName":"aaa@foo.com","submitterId":"8f51e348-1fa6-4194-8d13-8e0368e3298a","artifactId":"Livy","jobType":"SparkBatch","result":"Succeeded","schedulerInfo":{"submittedAt":"2020-01-10T02:36:03.5192707+00:00","scheduledAt":"2020-01-10T02:36:04.8237808+00:00","endedAt":"2020-01-10T02:41:21.1219891+00:00","cancellationRequestedAt":null,"currentState":"Ended"},"pluginInfo":{"preparationStartedAt":"2020-01-10T02:36:04.8373477+00:00","resourceAcquisitionStartedAt":"2020-01-10T02:36:05.330088+00:00","submissionStartedAt":"2020-01-10T02:37:52.5155982+00:00","monitoringStartedAt":"2020-01-10T02:38:45.9415399+00:00","cleanupStartedAt":"2020-01-10T02:41:20.4156114+00:00","currentState":"Ended"},"tags":{},"id":12,"appId":"application_1578623882988_0001","appInfo":{"driverLogUrl":"http://ce76a4b8792e4537a006d10d8954727a0370ec92478:8042/node/containerlogs/container_1578623882988_0001_01_000001/trusted-service-user","sparkUiUrl":"http://ce76a4b8792e4537a006d10d8954727a04390958445:8088/proxy/application_1578623882988_0001/"},"state":"success","log":["\t - tracking URL: http://ce76a4b8792e4537a006d10d8954727a04390958445:8088/proxy/application_1578623882988_0001/","\t - user: trusted-service-user","20/01/10 02:39:19 INFO ShutdownHookManager: Shutdown - hook called","20/01/10 02:39:19 INFO ShutdownHookManager: Deleting directory - /tmp/spark-5ecda28f-e4b9-4a78-8c2d-da3048be2f0e","20/01/10 02:39:19 INFO ShutdownHookManager: - Deleting directory /tmp/spark-0164a65c-591c-460d-ab19-c67ae8b411d6","20/01/10 - 02:39:19 INFO MetricsSystemImpl: Stopping azure-file-system metrics system...","20/01/10 - 02:39:19 INFO MetricsSystemImpl: azure-file-system metrics system stopped.","20/01/10 - 02:39:19 INFO MetricsSystemImpl: azure-file-system metrics system shutdown - complete.","\nstderr: ","\nYARN Diagnostics: "]},{"livyInfo":{"notStartedAt":null,"startingAt":"2020-01-10T03:38:29.2385918+00:00","runningAt":"2020-01-10T03:39:35.4499932+00:00","deadAt":null,"successAt":"2020-01-10T03:40:19.5803755+00:00","killedAt":null,"recoveringAt":null,"currentState":"success","jobCreationRequest":{"name":"WordCount_Java","file":"abfss://aaa@foo.com/java/wordcount/wordcount.jar","className":"WordCount","args":["abfss://aaa@foo.com/java/wordcount/shakespeare.txt","abfss://aaa@foo.com/java/wordcount/result/"],"conf":{},"driverMemory":"4g","driverCores":4,"executorMemory":"4g","executorCores":4,"numExecutors":2}},"name":"WordCount_Java","workspaceName":"testsynapseworkspace","computeName":"testsparkpool","sparkPoolName":"testsparkpool","submitterName":"aaa@foo.com","submitterId":"8f51e348-1fa6-4194-8d13-8e0368e3298a","artifactId":"Livy","jobType":"SparkBatch","result":"Succeeded","schedulerInfo":{"submittedAt":"2020-01-10T03:35:41.0317404+00:00","scheduledAt":"2020-01-10T03:35:42.1385977+00:00","endedAt":"2020-01-10T03:40:37.5042513+00:00","cancellationRequestedAt":null,"currentState":"Ended"},"pluginInfo":{"preparationStartedAt":"2020-01-10T03:35:42.1521105+00:00","resourceAcquisitionStartedAt":"2020-01-10T03:35:50.939074+00:00","submissionStartedAt":"2020-01-10T03:38:05.8559022+00:00","monitoringStartedAt":"2020-01-10T03:38:29.2694764+00:00","cleanupStartedAt":"2020-01-10T03:40:19.5962197+00:00","currentState":"Ended"},"tags":{},"id":13,"appId":"application_1578627484298_0001","appInfo":{"driverLogUrl":"http://9b16e3f9ff074564bf178da319ac14c60157d104065:8042/node/containerlogs/container_1578627484298_0001_01_000001/trusted-service-user","sparkUiUrl":"http://9b16e3f9ff074564bf178da319ac14c6021e9217492:8088/proxy/application_1578627484298_0001/"},"state":"success","log":["\t - tracking URL: http://9b16e3f9ff074564bf178da319ac14c6021e9217492:8088/proxy/application_1578627484298_0001/","\t - user: trusted-service-user","20/01/10 03:38:58 INFO ShutdownHookManager: Shutdown - hook called","20/01/10 03:38:58 INFO ShutdownHookManager: Deleting directory - /tmp/spark-3510c3ec-19c2-4675-8431-9d4c4b2e3cf5","20/01/10 03:38:58 INFO ShutdownHookManager: - Deleting directory /tmp/spark-925b68f4-72d8-4ac7-b62d-f7005eb5adaf","20/01/10 - 03:38:58 INFO MetricsSystemImpl: Stopping azure-file-system metrics system...","20/01/10 - 03:38:58 INFO MetricsSystemImpl: azure-file-system metrics system stopped.","20/01/10 - 03:38:58 INFO MetricsSystemImpl: azure-file-system metrics system shutdown - complete.","\nstderr: ","\nYARN Diagnostics: "]},{"livyInfo":{"notStartedAt":null,"startingAt":"2020-01-10T03:40:36.7442068+00:00","runningAt":"2020-01-10T03:41:20.8918068+00:00","deadAt":null,"successAt":"2020-01-10T03:42:05.043699+00:00","killedAt":null,"recoveringAt":null,"currentState":"success","jobCreationRequest":{"name":"WordCount_Java","file":"abfss://aaa@foo.com/java/wordcount/wordcount.jar","className":"WordCount","args":["abfss://aaa@foo.com/java/wordcount/shakespeare.txt","abfss://aaa@foo.com/java/wordcount/result/"],"conf":{},"driverMemory":"4g","driverCores":4,"executorMemory":"4g","executorCores":4,"numExecutors":2}},"name":"WordCount_Java","workspaceName":"testsynapseworkspace","computeName":"testsparkpool","sparkPoolName":"testsparkpool","submitterName":"aaa@foo.com","submitterId":"8f51e348-1fa6-4194-8d13-8e0368e3298a","artifactId":"Livy","jobType":"SparkBatch","result":"Succeeded","schedulerInfo":{"submittedAt":"2020-01-10T03:40:32.5418704+00:00","scheduledAt":"2020-01-10T03:40:33.6864167+00:00","endedAt":"2020-01-10T03:42:05.5656043+00:00","cancellationRequestedAt":null,"currentState":"Ended"},"pluginInfo":{"preparationStartedAt":"2020-01-10T03:40:33.6991753+00:00","resourceAcquisitionStartedAt":"2020-01-10T03:40:33.894599+00:00","submissionStartedAt":"2020-01-10T03:40:36.6783498+00:00","monitoringStartedAt":"2020-01-10T03:40:36.7605746+00:00","cleanupStartedAt":"2020-01-10T03:42:05.059249+00:00","currentState":"Ended"},"tags":{},"id":14,"appId":"application_1578627484298_0002","appInfo":{"driverLogUrl":"http://9b16e3f9ff074564bf178da319ac14c6013fa834371:8042/node/containerlogs/container_1578627484298_0002_01_000001/trusted-service-user","sparkUiUrl":"http://9b16e3f9ff074564bf178da319ac14c6021e9217492:8088/proxy/application_1578627484298_0002/"},"state":"success","log":["\t - tracking URL: http://9b16e3f9ff074564bf178da319ac14c6021e9217492:8088/proxy/application_1578627484298_0002/","\t - user: trusted-service-user","20/01/10 03:40:45 INFO ShutdownHookManager: Shutdown - hook called","20/01/10 03:40:45 INFO ShutdownHookManager: Deleting directory - /tmp/spark-05372e52-3f31-435b-9a89-f0e59d031147","20/01/10 03:40:45 INFO ShutdownHookManager: - Deleting directory /tmp/spark-6d758165-8230-4281-b7ba-9b8fef9d6d19","20/01/10 - 03:40:45 INFO MetricsSystemImpl: Stopping azure-file-system metrics system...","20/01/10 - 03:40:45 INFO MetricsSystemImpl: azure-file-system metrics system stopped.","20/01/10 - 03:40:45 INFO MetricsSystemImpl: azure-file-system metrics system shutdown - complete.","\nstderr: ","\nYARN Diagnostics: "]},{"livyInfo":{"notStartedAt":null,"startingAt":"2020-01-10T04:27:23.9757023+00:00","runningAt":"2020-01-10T04:28:52.2822789+00:00","deadAt":null,"successAt":"2020-01-10T04:29:36.4419439+00:00","killedAt":null,"recoveringAt":null,"currentState":"success","jobCreationRequest":{"name":"WordCount_Java","file":"abfss://aaa@foo.com/java/wordcount/wordcount.jar","className":"WordCount","args":["abfss://aaa@foo.com/java/wordcount/shakespeare.txt","abfss://aaa@foo.com/java/wordcount/result/"],"conf":{},"driverMemory":"4g","driverCores":4,"executorMemory":"4g","executorCores":4,"numExecutors":2}},"name":"WordCount_Java","workspaceName":"testsynapseworkspace","computeName":"testsparkpool","sparkPoolName":"testsparkpool","submitterName":"aaa@foo.com","submitterId":"8f51e348-1fa6-4194-8d13-8e0368e3298a","artifactId":"Livy","jobType":"SparkBatch","result":"Succeeded","schedulerInfo":{"submittedAt":"2020-01-10T04:25:25.8587068+00:00","scheduledAt":"2020-01-10T04:25:27.0358801+00:00","endedAt":"2020-01-10T04:29:37.0862105+00:00","cancellationRequestedAt":null,"currentState":"Ended"},"pluginInfo":{"preparationStartedAt":"2020-01-10T04:25:27.0502376+00:00","resourceAcquisitionStartedAt":"2020-01-10T04:25:27.3518291+00:00","submissionStartedAt":"2020-01-10T04:27:13.436335+00:00","monitoringStartedAt":"2020-01-10T04:27:24.0000169+00:00","cleanupStartedAt":"2020-01-10T04:29:36.4592291+00:00","currentState":"Ended"},"tags":{},"id":15,"appId":"application_1578630430582_0001","appInfo":{"driverLogUrl":"http://8e6f1bbb1f0d402ead227bfb3f9541ce029e8073802:8042/node/containerlogs/container_1578630430582_0001_01_000001/trusted-service-user","sparkUiUrl":"http://8e6f1bbb1f0d402ead227bfb3f9541ce019a8a47711:8088/proxy/application_1578630430582_0001/"},"state":"success","log":["\t - tracking URL: http://8e6f1bbb1f0d402ead227bfb3f9541ce019a8a47711:8088/proxy/application_1578630430582_0001/","\t - user: trusted-service-user","20/01/10 04:28:11 INFO ShutdownHookManager: Shutdown - hook called","20/01/10 04:28:11 INFO ShutdownHookManager: Deleting directory - /tmp/spark-e7390b6e-3c89-4d15-8504-dfc0713dbac2","20/01/10 04:28:11 INFO ShutdownHookManager: - Deleting directory /tmp/spark-1aa9ff81-602d-483e-8523-72db6bf69662","20/01/10 - 04:28:11 INFO MetricsSystemImpl: Stopping azure-file-system metrics system...","20/01/10 - 04:28:11 INFO MetricsSystemImpl: azure-file-system metrics system stopped.","20/01/10 - 04:28:11 INFO MetricsSystemImpl: azure-file-system metrics system shutdown - complete.","\nstderr: ","\nYARN Diagnostics: "]},{"livyInfo":{"notStartedAt":null,"startingAt":"2020-01-13T02:48:36.8904406+00:00","runningAt":"2020-01-13T02:49:54.1454521+00:00","deadAt":null,"successAt":"2020-01-13T02:50:27.2071468+00:00","killedAt":null,"recoveringAt":null,"currentState":"success","jobCreationRequest":{"name":"WordCount_Java","file":"abfss://aaa@foo.com/java/wordcount/wordcount.jar","className":"WordCount","args":["abfss://aaa@foo.com/java/wordcount/shakespeare.txt","abfss://aaa@foo.com/java/wordcount/result/"],"conf":{},"driverMemory":"4g","driverCores":4,"executorMemory":"4g","executorCores":4,"numExecutors":2}},"name":"WordCount_Java","workspaceName":"testsynapseworkspace","computeName":"testsparkpool","sparkPoolName":"testsparkpool","submitterName":"aaa@foo.com","submitterId":"8f51e348-1fa6-4194-8d13-8e0368e3298a","artifactId":"Livy","jobType":"SparkBatch","result":"Succeeded","schedulerInfo":{"submittedAt":"2020-01-13T02:47:05.5958128+00:00","scheduledAt":"2020-01-13T02:47:06.7547099+00:00","endedAt":"2020-01-13T02:50:27.949451+00:00","cancellationRequestedAt":null,"currentState":"Ended"},"pluginInfo":{"preparationStartedAt":"2020-01-13T02:47:06.7680533+00:00","resourceAcquisitionStartedAt":"2020-01-13T02:47:07.0124685+00:00","submissionStartedAt":"2020-01-13T02:48:36.0061327+00:00","monitoringStartedAt":"2020-01-13T02:48:36.9485387+00:00","cleanupStartedAt":"2020-01-13T02:50:27.2555545+00:00","currentState":"Ended"},"tags":{},"id":16,"appId":"application_1578883706606_0001","appInfo":{"driverLogUrl":"http://e3c1df4b4aa94ae6b7e5d19359b699fe041b3e01222:8042/node/containerlogs/container_1578883706606_0001_01_000001/trusted-service-user","sparkUiUrl":"http://e3c1df4b4aa94ae6b7e5d19359b699fe0302c276274:8088/proxy/application_1578883706606_0001/"},"state":"success","log":["\t - tracking URL: http://e3c1df4b4aa94ae6b7e5d19359b699fe0302c276274:8088/proxy/application_1578883706606_0001/","\t - user: trusted-service-user","20/01/13 02:49:38 INFO ShutdownHookManager: Shutdown - hook called","20/01/13 02:49:38 INFO ShutdownHookManager: Deleting directory - /tmp/spark-2c25f086-79bf-4db5-98ef-a50ec241447f","20/01/13 02:49:38 INFO ShutdownHookManager: - Deleting directory /tmp/spark-6e4d32f6-a1ab-4b78-967e-068b74920ad7","20/01/13 - 02:49:38 INFO MetricsSystemImpl: Stopping azure-file-system metrics system...","20/01/13 - 02:49:38 INFO MetricsSystemImpl: azure-file-system metrics system stopped.","20/01/13 - 02:49:38 INFO MetricsSystemImpl: azure-file-system metrics system shutdown - complete.","\nstderr: ","\nYARN Diagnostics: "]},{"livyInfo":{"notStartedAt":null,"startingAt":"2020-01-14T08:05:56.2723695+00:00","runningAt":null,"deadAt":"2020-01-14T08:08:08.7536371+00:00","successAt":null,"killedAt":null,"recoveringAt":null,"currentState":"dead","jobCreationRequest":{"name":"WordCount_Dotnet","file":"abfss://aaa@foo.com/dotnet/wordcount/microsoft-spark-2.4.x-0.6.0.jar","className":"org.apache.spark.deploy.dotnet.DotnetRunner","args":["abfss://aaa@foo.com/dotnet/wordcount/wordcount.zip","abfss://aaa@foo.com/dotnet/wordcount/shakespeare.txt","abfss://aaa@foo.com/dotnet/wordcount/result/"],"conf":{},"driverMemory":"4g","driverCores":4,"executorMemory":"4g","executorCores":4,"numExecutors":2}},"name":"WordCount_Dotnet","workspaceName":"testsynapseworkspace","computeName":"testsparkpool","sparkPoolName":"testsparkpool","submitterName":"aaa@foo.com","submitterId":"8f51e348-1fa6-4194-8d13-8e0368e3298a","artifactId":"Livy","jobType":"SparkBatch","result":"Failed","schedulerInfo":{"submittedAt":"2020-01-14T08:03:16.2526911+00:00","scheduledAt":"2020-01-14T08:03:17.3380957+00:00","endedAt":"2020-01-14T08:08:09.6745907+00:00","cancellationRequestedAt":null,"currentState":"Ended"},"pluginInfo":{"preparationStartedAt":"2020-01-14T08:03:17.3526315+00:00","resourceAcquisitionStartedAt":"2020-01-14T08:03:17.6777575+00:00","submissionStartedAt":"2020-01-14T08:05:12.4702664+00:00","monitoringStartedAt":"2020-01-14T08:05:56.3294055+00:00","cleanupStartedAt":"2020-01-14T08:08:08.7948883+00:00","currentState":"Ended"},"errorInfo":[{"message":"[plugins.testsynapseworkspace.testsparkpool.17] - [Monitoring] Livy Endpoint=[http://171.0.6.7:8998/]. Livy Id=[0] Job failed - during run time with state=[dead].","errorCode":"LIVY_JOB_FAILED","source":"Dependency"}],"tags":{},"id":17,"appId":"application_1578989125483_0001","appInfo":{"driverLogUrl":"http://1657321302a648dcb31a6629c8d3bc9b040d6388685:8042/node/containerlogs/container_1578989125483_0001_02_000001/trusted-service-user","sparkUiUrl":"http://1657321302a648dcb31a6629c8d3bc9b0230e607756:8088/cluster/app/application_1578989125483_0001"},"state":"dead","log":["20/01/14 - 08:07:59 WARN AzureFileSystemThreadPoolExecutor: Disabling threads for Delete - operation as thread count 0 is <= 1","20/01/14 08:07:59 INFO AzureFileSystemThreadPoolExecutor: - Time taken for Delete operation is: 13 ms with threads: 0","20/01/14 08:07:59 - INFO ShutdownHookManager: Shutdown hook called","20/01/14 08:07:59 INFO MetricsSystemImpl: - Stopping azure-file-system metrics system...","20/01/14 08:07:59 INFO MetricsSystemImpl: - azure-file-system metrics system stopped.","20/01/14 08:07:59 INFO MetricsSystemImpl: - azure-file-system metrics system shutdown complete.","","","For more detailed - output, check the application tracking page: http://1657321302a648dcb31a6629c8d3bc9b0230e607756:8088/cluster/app/application_1578989125483_0001 - Then click on links to logs of each attempt.",". Failing the application."]},{"livyInfo":{"notStartedAt":null,"startingAt":"2020-01-14T08:23:04.8045606+00:00","runningAt":"2020-01-14T08:24:10.9877487+00:00","deadAt":null,"successAt":"2020-01-14T08:25:28.1879416+00:00","killedAt":null,"recoveringAt":null,"currentState":"success","jobCreationRequest":{"name":"WordCount_Dotnet","file":"abfss://aaa@foo.com/dotnet/wordcount/microsoft-spark-2.4.x-0.6.0.jar","className":"org.apache.spark.deploy.dotnet.DotnetRunner","args":["abfss://aaa@foo.com/dotnet/wordcount/wordcount.zip","WordCount","abfss://aaa@foo.com/dotnet/wordcount/shakespeare.txt","abfss://aaa@foo.com/dotnet/wordcount/result"],"conf":{},"driverMemory":"4g","driverCores":4,"executorMemory":"4g","executorCores":4,"numExecutors":2}},"name":"WordCount_Dotnet","workspaceName":"testsynapseworkspace","computeName":"testsparkpool","sparkPoolName":"testsparkpool","submitterName":"aaa@foo.com","submitterId":"8f51e348-1fa6-4194-8d13-8e0368e3298a","artifactId":"Livy","jobType":"SparkBatch","result":"Succeeded","schedulerInfo":{"submittedAt":"2020-01-14T08:23:00.5240826+00:00","scheduledAt":"2020-01-14T08:23:01.6456144+00:00","endedAt":"2020-01-14T08:25:28.9790621+00:00","cancellationRequestedAt":null,"currentState":"Ended"},"pluginInfo":{"preparationStartedAt":"2020-01-14T08:23:01.65923+00:00","resourceAcquisitionStartedAt":"2020-01-14T08:23:01.8972284+00:00","submissionStartedAt":"2020-01-14T08:23:04.6997445+00:00","monitoringStartedAt":"2020-01-14T08:23:04.8193186+00:00","cleanupStartedAt":"2020-01-14T08:25:28.2053071+00:00","currentState":"Ended"},"tags":{},"id":18,"appId":"application_1578989125483_0002","appInfo":{"driverLogUrl":"http://1657321302a648dcb31a6629c8d3bc9b012e5f40483:8042/node/containerlogs/container_1578989125483_0002_01_000001/trusted-service-user","sparkUiUrl":"http://1657321302a648dcb31a6629c8d3bc9b0230e607756:8088/proxy/application_1578989125483_0002/"},"state":"success","log":["\t - tracking URL: http://1657321302a648dcb31a6629c8d3bc9b0230e607756:8088/proxy/application_1578989125483_0002/","\t - user: trusted-service-user","20/01/14 08:23:13 INFO ShutdownHookManager: Shutdown - hook called","20/01/14 08:23:13 INFO ShutdownHookManager: Deleting directory - /tmp/spark-07aae8db-d0be-4c05-803a-474c353ffeb4","20/01/14 08:23:13 INFO ShutdownHookManager: - Deleting directory /tmp/spark-dd484af5-ca45-44d9-adac-609c2a76e34e","20/01/14 - 08:23:13 INFO MetricsSystemImpl: Stopping azure-file-system metrics system...","20/01/14 - 08:23:13 INFO MetricsSystemImpl: azure-file-system metrics system stopped.","20/01/14 - 08:23:13 INFO MetricsSystemImpl: azure-file-system metrics system shutdown - complete.","\nstderr: ","\nYARN Diagnostics: "]},{"livyInfo":{"notStartedAt":null,"startingAt":"2020-01-15T01:23:50.8375423+00:00","runningAt":"2020-01-15T01:24:57.1362447+00:00","deadAt":null,"successAt":"2020-01-15T01:26:14.351845+00:00","killedAt":null,"recoveringAt":null,"currentState":"success","jobCreationRequest":{"name":"WordCount_Dotnet_cli","file":"abfss://aaa@foo.com/dotnet/wordcount/microsoft-spark-2.4.x-0.6.0.jar","className":"org.apache.spark.deploy.dotnet.DotnetRunner","args":["abfss://aaa@foo.com/dotnet/wordcount/wordcount.zip","WordCount","abfss://aaa@foo.com/dotnet/wordcount/shakespeare.txt","abfss://aaa@foo.com/dotnet/wordcount/result"],"conf":{},"driverMemory":"4g","driverCores":4,"executorMemory":"4g","executorCores":4,"numExecutors":2}},"name":"WordCount_Dotnet_cli","workspaceName":"testsynapseworkspace","computeName":"testsparkpool","sparkPoolName":"testsparkpool","submitterName":"aaa@foo.com","submitterId":"8f51e348-1fa6-4194-8d13-8e0368e3298a","artifactId":"Livy","jobType":"SparkBatch","result":"Succeeded","schedulerInfo":{"submittedAt":"2020-01-15T01:21:44.1055566+00:00","scheduledAt":"2020-01-15T01:21:45.3573103+00:00","endedAt":"2020-01-15T01:26:15.0975636+00:00","cancellationRequestedAt":null,"currentState":"Ended"},"pluginInfo":{"preparationStartedAt":"2020-01-15T01:21:45.3717089+00:00","resourceAcquisitionStartedAt":"2020-01-15T01:21:45.6917453+00:00","submissionStartedAt":"2020-01-15T01:23:37.8659497+00:00","monitoringStartedAt":"2020-01-15T01:23:50.8553888+00:00","cleanupStartedAt":"2020-01-15T01:26:14.3683065+00:00","currentState":"Ended"},"tags":{},"id":19,"appId":"application_1579051421404_0001","appInfo":{"driverLogUrl":"http://1b69b0a4ed994f7ea95f9d85f63a3083009f3081028:8042/node/containerlogs/container_1579051421404_0001_01_000001/trusted-service-user","sparkUiUrl":"http://1b69b0a4ed994f7ea95f9d85f63a308302603012606:8088/proxy/application_1579051421404_0001/"},"state":"success","log":["\t - tracking URL: http://1b69b0a4ed994f7ea95f9d85f63a308302603012606:8088/proxy/application_1579051421404_0001/","\t - user: trusted-service-user","20/01/15 01:24:14 INFO ShutdownHookManager: Shutdown - hook called","20/01/15 01:24:14 INFO ShutdownHookManager: Deleting directory - /tmp/spark-e9350640-4f8f-4f9d-8b8b-6dff3a6104fa","20/01/15 01:24:14 INFO ShutdownHookManager: - Deleting directory /tmp/spark-dc37dd45-d38e-4c84-b821-602d8c111e23","20/01/15 - 01:24:14 INFO MetricsSystemImpl: Stopping azure-file-system metrics system...","20/01/15 - 01:24:14 INFO MetricsSystemImpl: azure-file-system metrics system stopped.","20/01/15 - 01:24:14 INFO MetricsSystemImpl: azure-file-system metrics system shutdown - complete.","\nstderr: ","\nYARN Diagnostics: "]},{"livyInfo":{"notStartedAt":null,"startingAt":"2020-01-16T02:11:09.1528283+00:00","runningAt":"2020-01-16T02:12:15.3521866+00:00","deadAt":null,"successAt":"2020-01-16T02:12:48.4623121+00:00","killedAt":null,"recoveringAt":null,"currentState":"success","jobCreationRequest":{"name":"WordCount_Dotnet","file":"abfss://aaa@foo.com/dotnet/wordcount/microsoft-spark-2.4.x-0.6.0.jar","className":"org.apache.spark.deploy.dotnet.DotnetRunner","args":["abfss://aaa@foo.com/dotnet/wordcount/wordcount.zip","WordCount","abfss://aaa@foo.com/dotnet/wordcount/shakespeare.txt","abfss://aaa@foo.com/dotnet/wordcount/result"],"conf":{},"driverMemory":"4g","driverCores":4,"executorMemory":"4g","executorCores":4,"numExecutors":2}},"name":"WordCount_Dotnet","workspaceName":"testsynapseworkspace","computeName":"testsparkpool","sparkPoolName":"testsparkpool","submitterName":"aaa@foo.com","submitterId":"8f51e348-1fa6-4194-8d13-8e0368e3298a","artifactId":"Livy","jobType":"SparkBatch","result":"Succeeded","schedulerInfo":{"submittedAt":"2020-01-16T02:08:56.7032948+00:00","scheduledAt":"2020-01-16T02:08:57.8157792+00:00","endedAt":"2020-01-16T02:12:49.0505851+00:00","cancellationRequestedAt":null,"currentState":"Ended"},"pluginInfo":{"preparationStartedAt":"2020-01-16T02:08:57.8407645+00:00","resourceAcquisitionStartedAt":"2020-01-16T02:08:58.0167402+00:00","submissionStartedAt":"2020-01-16T02:11:08.5234297+00:00","monitoringStartedAt":"2020-01-16T02:11:09.169573+00:00","cleanupStartedAt":"2020-01-16T02:12:48.4771889+00:00","currentState":"Ended"},"tags":{},"id":20,"appId":"application_1579140649715_0001","appInfo":{"driverLogUrl":"http://c5bf52cc122a4e8b8e42bb633501e6770007eb66248:8042/node/containerlogs/container_1579140649715_0001_01_000001/trusted-service-user","sparkUiUrl":"http://c5bf52cc122a4e8b8e42bb633501e67704015827933:8088/proxy/application_1579140649715_0001/"},"state":"success","log":["\t - tracking URL: http://c5bf52cc122a4e8b8e42bb633501e67704015827933:8088/proxy/application_1579140649715_0001/","\t - user: trusted-service-user","20/01/16 02:11:50 INFO ShutdownHookManager: Shutdown - hook called","20/01/16 02:11:50 INFO ShutdownHookManager: Deleting directory - /tmp/spark-dcd3cb18-f03f-4ac5-967f-51fa07df172f","20/01/16 02:11:50 INFO ShutdownHookManager: - Deleting directory /tmp/spark-68628a62-1cf0-4f58-bb6b-22fc97b2cc0b","20/01/16 - 02:11:51 INFO MetricsSystemImpl: Stopping azure-file-system metrics system...","20/01/16 - 02:11:51 INFO MetricsSystemImpl: azure-file-system metrics system stopped.","20/01/16 - 02:11:51 INFO MetricsSystemImpl: azure-file-system metrics system shutdown - complete.","\nstderr: ","\nYARN Diagnostics: "]},{"livyInfo":{"notStartedAt":null,"startingAt":"2020-01-16T02:21:08.8565042+00:00","runningAt":"2020-01-16T02:21:41.9836978+00:00","deadAt":null,"successAt":"2020-01-16T02:22:15.0791064+00:00","killedAt":null,"recoveringAt":null,"currentState":"success","jobCreationRequest":{"name":"WordCount_Dotnet","file":"abfss://aaa@foo.com/dotnet/wordcount/microsoft-spark-2.4.x-0.6.0.jar","className":"org.apache.spark.deploy.dotnet.DotnetRunner","args":["abfss://aaa@foo.com/dotnet/wordcount/wordcount.zip","WordCount","abfss://aaa@foo.com/dotnet/wordcount/shakespeare.txt","abfss://aaa@foo.com/dotnet/wordcount/result"],"conf":{},"driverMemory":"4g","driverCores":4,"executorMemory":"4g","executorCores":4,"numExecutors":2}},"name":"WordCount_Dotnet","workspaceName":"testsynapseworkspace","computeName":"testsparkpool","sparkPoolName":"testsparkpool","submitterName":"aaa@foo.com","submitterId":"8f51e348-1fa6-4194-8d13-8e0368e3298a","artifactId":"Livy","jobType":"SparkBatch","result":"Succeeded","schedulerInfo":{"submittedAt":"2020-01-16T02:21:04.5602571+00:00","scheduledAt":"2020-01-16T02:21:05.5944531+00:00","endedAt":"2020-01-16T02:22:15.6683532+00:00","cancellationRequestedAt":null,"currentState":"Ended"},"pluginInfo":{"preparationStartedAt":"2020-01-16T02:21:05.606491+00:00","resourceAcquisitionStartedAt":"2020-01-16T02:21:05.802267+00:00","submissionStartedAt":"2020-01-16T02:21:08.7788972+00:00","monitoringStartedAt":"2020-01-16T02:21:08.8712008+00:00","cleanupStartedAt":"2020-01-16T02:22:15.0960969+00:00","currentState":"Ended"},"tags":{},"id":21,"appId":"application_1579140649715_0002","appInfo":{"driverLogUrl":"http://c5bf52cc122a4e8b8e42bb633501e67702199d05116:8042/node/containerlogs/container_1579140649715_0002_01_000001/trusted-service-user","sparkUiUrl":"http://c5bf52cc122a4e8b8e42bb633501e67704015827933:8088/proxy/application_1579140649715_0002/"},"state":"success","log":["\t - tracking URL: http://c5bf52cc122a4e8b8e42bb633501e67704015827933:8088/proxy/application_1579140649715_0002/","\t - user: trusted-service-user","20/01/16 02:21:17 INFO ShutdownHookManager: Shutdown - hook called","20/01/16 02:21:17 INFO ShutdownHookManager: Deleting directory - /tmp/spark-acb065c5-b5a0-43f5-9bfc-13a0dc232598","20/01/16 02:21:17 INFO ShutdownHookManager: - Deleting directory /tmp/spark-2f39dc65-0986-4470-ac8c-e41754627e0e","20/01/16 - 02:21:17 INFO MetricsSystemImpl: Stopping azure-file-system metrics system...","20/01/16 - 02:21:17 INFO MetricsSystemImpl: azure-file-system metrics system stopped.","20/01/16 - 02:21:17 INFO MetricsSystemImpl: azure-file-system metrics system shutdown - complete.","\nstderr: ","\nYARN Diagnostics: "]},{"livyInfo":{"notStartedAt":null,"startingAt":"2020-01-16T02:23:58.0339139+00:00","runningAt":"2020-01-16T02:24:31.0910495+00:00","deadAt":null,"successAt":"2020-01-16T02:25:04.1861059+00:00","killedAt":null,"recoveringAt":null,"currentState":"success","jobCreationRequest":{"name":"WordCount_Dotnet","file":"abfss://aaa@foo.com/dotnet/wordcount/microsoft-spark-2.4.x-0.6.0.jar","className":"org.apache.spark.deploy.dotnet.DotnetRunner","args":["abfss://aaa@foo.com/dotnet/wordcount/wordcount.zip","WordCount","abfss://aaa@foo.com/dotnet/wordcount/shakespeare.txt","abfss://aaa@foo.com/dotnet/wordcount/result"],"conf":{},"driverMemory":"4g","driverCores":4,"executorMemory":"4g","executorCores":4,"numExecutors":2}},"name":"WordCount_Dotnet","workspaceName":"testsynapseworkspace","computeName":"testsparkpool","sparkPoolName":"testsparkpool","submitterName":"aaa@foo.com","submitterId":"8f51e348-1fa6-4194-8d13-8e0368e3298a","artifactId":"Livy","jobType":"SparkBatch","result":"Succeeded","schedulerInfo":{"submittedAt":"2020-01-16T02:23:53.8543022+00:00","scheduledAt":"2020-01-16T02:23:54.9361073+00:00","endedAt":"2020-01-16T02:25:05.076203+00:00","cancellationRequestedAt":null,"currentState":"Ended"},"pluginInfo":{"preparationStartedAt":"2020-01-16T02:23:54.9486634+00:00","resourceAcquisitionStartedAt":"2020-01-16T02:23:55.2518511+00:00","submissionStartedAt":"2020-01-16T02:23:57.9528435+00:00","monitoringStartedAt":"2020-01-16T02:23:58.0499442+00:00","cleanupStartedAt":"2020-01-16T02:25:04.2016484+00:00","currentState":"Ended"},"tags":{},"id":22,"appId":"application_1579140649715_0003","appInfo":{"driverLogUrl":"http://c5bf52cc122a4e8b8e42bb633501e67702712f69936:8042/node/containerlogs/container_1579140649715_0003_01_000001/trusted-service-user","sparkUiUrl":"http://c5bf52cc122a4e8b8e42bb633501e67704015827933:8088/proxy/application_1579140649715_0003/"},"state":"success","log":["\t - tracking URL: http://c5bf52cc122a4e8b8e42bb633501e67704015827933:8088/proxy/application_1579140649715_0003/","\t - user: trusted-service-user","20/01/16 02:24:06 INFO ShutdownHookManager: Shutdown - hook called","20/01/16 02:24:06 INFO ShutdownHookManager: Deleting directory - /tmp/spark-91ae2c9d-68e6-463f-8777-3cb4943600e8","20/01/16 02:24:06 INFO ShutdownHookManager: - Deleting directory /tmp/spark-7bb6fe6c-9aed-4d01-a39a-e21b9a04cf8a","20/01/16 - 02:24:06 INFO MetricsSystemImpl: Stopping azure-file-system metrics system...","20/01/16 - 02:24:06 INFO MetricsSystemImpl: azure-file-system metrics system stopped.","20/01/16 - 02:24:06 INFO MetricsSystemImpl: azure-file-system metrics system shutdown - complete.","\nstderr: ","\nYARN Diagnostics: "]},{"livyInfo":{"notStartedAt":null,"startingAt":"2020-01-16T09:39:38.6049357+00:00","runningAt":"2020-01-16T09:41:17.8427957+00:00","deadAt":null,"successAt":"2020-01-16T09:42:12.9407211+00:00","killedAt":null,"recoveringAt":null,"currentState":"success","jobCreationRequest":{"name":"batch_job_from_cli","file":"abfss://aaa@foo.com/java/wordcount/wordcount.jar","className":"WordCount","args":["abfss://aaa@foo.com/java/wordcount/shakespeare.txt","abfss://aaa@foo.com/java/wordcount/result/"],"conf":{},"driverMemory":"4g","driverCores":4,"executorMemory":"4g","executorCores":4,"numExecutors":2}},"name":"batch_job_from_cli","workspaceName":"testsynapseworkspace","computeName":"testsparkpool","sparkPoolName":"testsparkpool","submitterId":"ca728363-7297-4e35-a9c3-c6400837883e","artifactId":"Livy","jobType":"SparkBatch","result":"Succeeded","schedulerInfo":{"submittedAt":"2020-01-16T09:36:46.4661975+00:00","scheduledAt":"2020-01-16T09:36:47.6482388+00:00","endedAt":"2020-01-16T09:42:13.1288868+00:00","cancellationRequestedAt":null,"currentState":"Ended"},"pluginInfo":{"preparationStartedAt":"2020-01-16T09:36:47.6607577+00:00","resourceAcquisitionStartedAt":"2020-01-16T09:36:47.9764174+00:00","submissionStartedAt":"2020-01-16T09:39:01.2644498+00:00","monitoringStartedAt":"2020-01-16T09:39:38.6237587+00:00","cleanupStartedAt":"2020-01-16T09:42:12.9582947+00:00","currentState":"Ended"},"tags":{},"id":23,"appId":"application_1579167542444_0001","appInfo":{"driverLogUrl":"http://dff5fe0bf844496b896e527ae28480c20308c150511:8042/node/containerlogs/container_1579167542444_0001_01_000001/trusted-service-user","sparkUiUrl":"http://dff5fe0bf844496b896e527ae28480c20442ee71008:8088/proxy/application_1579167542444_0001/"},"state":"success","log":["\t - tracking URL: http://dff5fe0bf844496b896e527ae28480c20442ee71008:8088/proxy/application_1579167542444_0001/","\t - user: trusted-service-user","20/01/16 09:40:38 INFO ShutdownHookManager: Shutdown - hook called","20/01/16 09:40:38 INFO ShutdownHookManager: Deleting directory - /tmp/spark-64d62c83-9956-4af6-abce-3d4f6084e2f2","20/01/16 09:40:38 INFO ShutdownHookManager: - Deleting directory /tmp/spark-8191d5d3-b207-474d-969f-290f74820b3e","20/01/16 - 09:40:38 INFO MetricsSystemImpl: Stopping azure-file-system metrics system...","20/01/16 - 09:40:38 INFO MetricsSystemImpl: azure-file-system metrics system stopped.","20/01/16 - 09:40:38 INFO MetricsSystemImpl: azure-file-system metrics system shutdown - complete.","\nstderr: ","\nYARN Diagnostics: "]},{"livyInfo":{"notStartedAt":null,"startingAt":null,"runningAt":null,"deadAt":null,"successAt":null,"killedAt":null,"recoveringAt":null,"currentState":"error","jobCreationRequest":{"name":"batch_job_from_cli","file":"abfss://aaa@foo.com/java/wordcount/wordcount.jar","className":"WordCount","args":["abfss://aaa@foo.com/java/wordcount/shakespeare.txt","abfss://aaa@foo.com/java/wordcount/result/"],"conf":{},"driverMemory":"4g","driverCores":4,"executorMemory":"4g","executorCores":4,"numExecutors":2}},"name":"batch_job_from_cli","workspaceName":"testsynapseworkspace","computeName":"testsparkpool","sparkPoolName":"testsparkpool","submitterId":"ca728363-7297-4e35-a9c3-c6400837883e","artifactId":"Livy","jobType":"SparkBatch","result":"Cancelled","schedulerInfo":{"submittedAt":"2020-01-16T09:37:10.8498374+00:00","scheduledAt":null,"endedAt":"2020-01-16T09:37:11.1608828+00:00","cancellationRequestedAt":"2020-01-16T09:37:11.1606522+00:00","currentState":"Ended"},"pluginInfo":{"preparationStartedAt":null,"resourceAcquisitionStartedAt":null,"submissionStartedAt":null,"monitoringStartedAt":null,"cleanupStartedAt":"2020-01-16T09:37:11.1606589+00:00","currentState":"Ended"},"tags":{},"id":24,"appId":null,"appInfo":null,"state":"error","log":null},{"livyInfo":{"notStartedAt":null,"startingAt":null,"runningAt":null,"deadAt":null,"successAt":null,"killedAt":null,"recoveringAt":null,"currentState":"error","jobCreationRequest":{"name":"batch_job_from_cli","file":"abfss://aaa@foo.com/java/wordcount/wordcount.jar","className":"WordCount","args":["abfss://aaa@foo.com/java/wordcount/shakespeare.txt","abfss://aaa@foo.com/java/wordcount/result/"],"conf":{},"driverMemory":"4g","driverCores":4,"executorMemory":"4g","executorCores":4,"numExecutors":2}},"name":"batch_job_from_cli","workspaceName":"testsynapseworkspace","computeName":"testsparkpool","sparkPoolName":"testsparkpool","submitterId":"ca728363-7297-4e35-a9c3-c6400837883e","artifactId":"Livy","jobType":"SparkBatch","result":"Cancelled","schedulerInfo":{"submittedAt":"2020-01-16T09:37:54.1813823+00:00","scheduledAt":null,"endedAt":"2020-01-16T09:37:54.68767+00:00","cancellationRequestedAt":"2020-01-16T09:37:54.687406+00:00","currentState":"Ended"},"pluginInfo":{"preparationStartedAt":null,"resourceAcquisitionStartedAt":null,"submissionStartedAt":null,"monitoringStartedAt":null,"cleanupStartedAt":"2020-01-16T09:37:54.6874092+00:00","currentState":"Ended"},"tags":{},"id":25,"appId":null,"appInfo":null,"state":"error","log":null},{"livyInfo":{"notStartedAt":null,"startingAt":"2020-01-19T06:46:05.3847587+00:00","runningAt":"2020-01-19T06:46:49.5118675+00:00","deadAt":null,"successAt":"2020-01-19T06:47:33.6083779+00:00","killedAt":null,"recoveringAt":null,"currentState":"success","jobCreationRequest":{"name":"WordCount_Dotnet_with_local_jar","file":"local:///usr/hdp/current/spark2-client/jars/microsoft-spark.jar","className":"org.apache.spark.deploy.dotnet.DotnetRunner","args":["abfss://aaa@foo.com/dotnet/wordcount/wordcount.zip","WordCount","abfss://aaa@foo.com/dotnet/wordcount/shakespeare.txt","abfss://aaa@foo.com/dotnet/wordcount/result"],"conf":{},"driverMemory":"4g","driverCores":4,"executorMemory":"4g","executorCores":4,"numExecutors":2}},"name":"WordCount_Dotnet_with_local_jar","workspaceName":"testsynapseworkspace","computeName":"testsparkpool","sparkPoolName":"testsparkpool","submitterName":"aaa@foo.com","submitterId":"8f51e348-1fa6-4194-8d13-8e0368e3298a","artifactId":"Livy","jobType":"SparkBatch","result":"Succeeded","schedulerInfo":{"submittedAt":"2020-01-19T06:44:13.6466637+00:00","scheduledAt":"2020-01-19T06:44:13.7712602+00:00","endedAt":"2020-01-19T06:47:34.494125+00:00","cancellationRequestedAt":null,"currentState":"Ended"},"pluginInfo":{"preparationStartedAt":"2020-01-19T06:44:13.7855258+00:00","resourceAcquisitionStartedAt":"2020-01-19T06:44:14.0992483+00:00","submissionStartedAt":"2020-01-19T06:46:03.9490509+00:00","monitoringStartedAt":"2020-01-19T06:46:05.4036305+00:00","cleanupStartedAt":"2020-01-19T06:47:33.62279+00:00","currentState":"Ended"},"tags":{},"id":26,"appId":"application_1579416350077_0001","appInfo":{"driverLogUrl":"http://35c3cd39703446f7b5879bc0b8805f8c001de530649:8042/node/containerlogs/container_1579416350077_0001_01_000001/trusted-service-user","sparkUiUrl":"http://35c3cd39703446f7b5879bc0b8805f8c016c6327596:8088/proxy/application_1579416350077_0001/"},"state":"success","log":["\t - tracking URL: http://35c3cd39703446f7b5879bc0b8805f8c016c6327596:8088/proxy/application_1579416350077_0001/","\t - user: trusted-service-user","20/01/19 06:46:18 INFO ShutdownHookManager: Shutdown - hook called","20/01/19 06:46:18 INFO ShutdownHookManager: Deleting directory - /tmp/spark-de3d608d-97b0-4ca5-a6c5-b05df4a40e8e","20/01/19 06:46:18 INFO ShutdownHookManager: - Deleting directory /tmp/spark-ab19f399-5768-4010-a5aa-0257fc7ccc71","20/01/19 - 06:46:18 INFO MetricsSystemImpl: Stopping azure-file-system metrics system...","20/01/19 - 06:46:18 INFO MetricsSystemImpl: azure-file-system metrics system stopped.","20/01/19 - 06:46:18 INFO MetricsSystemImpl: azure-file-system metrics system shutdown - complete.","\nstderr: ","\nYARN Diagnostics: "]}]}' + string: '{"from":0,"total":1,"sessions":[{"livyInfo":{"notStartedAt":null,"startingAt":null,"runningAt":null,"deadAt":null,"successAt":null,"killedAt":null,"recoveringAt":null,"currentState":"not_started","jobCreationRequest":{"name":"WordCount_Java","file":"abfss://filesystem01@ywtestaccount.dfs.core.windows.net/wordcount/wordcount_2.11-0.1.jar","className":"WordCount","args":["[abfss://filesystem01@ywtestaccount.dfs.core.windows.net/wordcount/input/shakespeare.txt,","abfss://filesystem01@ywtestaccount.dfs.core.windows.net/wordcount/output/]"],"conf":{"spark.dynamicAllocation.maxExecutors":"18"},"driverMemory":"56g","driverCores":8,"executorMemory":"56g","executorCores":8,"numExecutors":4}},"name":"WordCount_Java","workspaceName":"yanjuntestws001","computeName":"sparkpooltest","sparkPoolName":"sparkpooltest","submitterName":"v-weishuzhao@microsoft.com","submitterId":"6a33200f-2b27-4fda-8567-b0c70d532441","artifactId":"Livy","jobType":"SparkBatch","result":"Uncertain","schedulerInfo":{"submittedAt":"2025-02-24T09:47:41.6861206+00:00","queuedAt":"2025-02-24T09:47:42.1202441+00:00","scheduledAt":null,"endedAt":null,"cancellationRequestedAt":null,"currentState":"Queued"},"pluginInfo":{"preparationStartedAt":null,"resourceAcquisitionStartedAt":null,"submissionStartedAt":null,"monitoringStartedAt":null,"cleanupStartedAt":null,"currentState":"Preparation"},"tags":{},"id":0,"appId":null,"appInfo":null,"state":"not_started","log":null,"registeredSources":null}]}' headers: + content-length: + - '1469' content-type: - application/json; charset=utf-8 date: - - Mon, 07 Jun 2021 09:13:48 GMT + - Mon, 24 Feb 2025 09:47:44 GMT server: - Kestrel Microsoft-HTTPAPI/2.0 strict-transport-security: - max-age=31536000; includeSubDomains - transfer-encoding: - - chunked x-ms-activity-id: - - 2bf38bab-ebf7-4aef-91c6-67ca1bda9f2e - x-ms-response-time-ms: - - '55' + - 6e3e5687-f16d-4fe5-b05d-a735223ca948 status: code: 200 message: OK @@ -326,9 +160,9 @@ interactions: Content-Length: - '0' User-Agent: - - azsdk-python-synapse/0.2.0 Python/3.7.4 (Windows-10-10.0.19041-SP0) + - azsdk-python-synapse-spark/0.7.0 Python/3.10.11 (Windows-10-10.0.26100-SP0) method: DELETE - uri: https://testsynapseworkspace.dev.azuresynapse.net/livyApi/versions/2019-11-01-preview/sparkPools/testsparkpool/batches/423 + uri: https://yanjuntestws001.dev.azuresynapse.net/livyApi/versions/2019-11-01-preview/sparkPools/sparkpooltest/batches/0 response: body: string: '' @@ -336,15 +170,15 @@ interactions: content-length: - '0' date: - - Mon, 07 Jun 2021 09:13:50 GMT + - Mon, 24 Feb 2025 09:47:45 GMT server: - Kestrel Microsoft-HTTPAPI/2.0 strict-transport-security: - max-age=31536000; includeSubDomains x-ms-activity-id: - - d1e01360-960a-4170-a7d8-2821bc008fc9 + - 1361c792-1957-48f3-8f8b-33166331fa4b x-ms-response-time-ms: - - '36' + - '33' status: code: 200 message: OK @@ -358,31 +192,31 @@ interactions: Connection: - keep-alive User-Agent: - - azsdk-python-synapse/0.2.0 Python/3.7.4 (Windows-10-10.0.19041-SP0) + - azsdk-python-synapse-spark/0.7.0 Python/3.10.11 (Windows-10-10.0.26100-SP0) method: GET - uri: https://testsynapseworkspace.dev.azuresynapse.net/livyApi/versions/2019-11-01-preview/sparkPools/testsparkpool/batches/423?detailed=true + uri: https://yanjuntestws001.dev.azuresynapse.net/livyApi/versions/2019-11-01-preview/sparkPools/sparkpooltest/batches/0?detailed=true response: body: - string: '{"livyInfo":{"notStartedAt":null,"startingAt":null,"runningAt":null,"deadAt":null,"successAt":null,"killedAt":null,"recoveringAt":null,"currentState":"error","jobCreationRequest":{"name":"WordCount_Java","file":"abfss://aaa@foo.com/java/wordcount/wordcount.jar","className":"WordCount","args":["[abfss://aaa@foo.com/java/wordcount/shakespeare.txt,","abfss://aaa@foo.com/java/wordcount/result/]"],"conf":{"spark.dynamicAllocation.maxExecutors":"18"},"driverMemory":"56g","driverCores":8,"executorMemory":"56g","executorCores":8,"numExecutors":2}},"name":"WordCount_Java","workspaceName":"testsynapseworkspace","computeName":"testsparkpool","sparkPoolName":"testsparkpool","submitterName":"aaa@foo.com","submitterId":"52379807-94ed-417a-9a37-b7751e3fb688","artifactId":"Livy","jobType":"SparkBatch","result":"Cancelled","schedulerInfo":{"submittedAt":"2021-06-07T09:13:40.7537681+00:00","scheduledAt":null,"endedAt":"2021-06-07T09:13:52.1076943+00:00","cancellationRequestedAt":"2021-06-07T09:13:50.888271+00:00","currentState":"Ended"},"pluginInfo":{"preparationStartedAt":null,"resourceAcquisitionStartedAt":null,"submissionStartedAt":null,"monitoringStartedAt":null,"cleanupStartedAt":"2021-06-07T09:13:50.8882747+00:00","currentState":"Ended"},"tags":{},"id":423,"appId":null,"appInfo":null,"state":"error","log":null}' + string: '{"livyInfo":{"notStartedAt":null,"startingAt":null,"runningAt":null,"deadAt":null,"successAt":null,"killedAt":null,"recoveringAt":null,"currentState":"not_started","jobCreationRequest":{"name":"WordCount_Java","file":"abfss://filesystem01@ywtestaccount.dfs.core.windows.net/wordcount/wordcount_2.11-0.1.jar","className":"WordCount","args":["[abfss://filesystem01@ywtestaccount.dfs.core.windows.net/wordcount/input/shakespeare.txt,","abfss://filesystem01@ywtestaccount.dfs.core.windows.net/wordcount/output/]"],"conf":{"spark.dynamicAllocation.maxExecutors":"18"},"driverMemory":"56g","driverCores":8,"executorMemory":"56g","executorCores":8,"numExecutors":4}},"name":"WordCount_Java","workspaceName":"yanjuntestws001","computeName":"sparkpooltest","sparkPoolName":"sparkpooltest","submitterName":"v-weishuzhao@microsoft.com","submitterId":"6a33200f-2b27-4fda-8567-b0c70d532441","artifactId":"Livy","jobType":"SparkBatch","result":"Cancelled","schedulerInfo":{"submittedAt":"2025-02-24T09:47:41.6861206+00:00","queuedAt":"2025-02-24T09:47:42.1202441+00:00","scheduledAt":null,"endedAt":"2025-02-24T09:47:49.5312196+00:00","cancellationRequestedAt":"2025-02-24T09:47:49.5294113+00:00","currentState":"Ended"},"pluginInfo":{"preparationStartedAt":null,"resourceAcquisitionStartedAt":null,"submissionStartedAt":null,"monitoringStartedAt":null,"cleanupStartedAt":"2025-02-24T09:47:49.5296112+00:00","currentState":"Ended"},"tags":{},"id":0,"appId":null,"appInfo":null,"state":"not_started","log":null,"registeredSources":null}' headers: + content-length: + - '1521' content-type: - application/json; charset=utf-8 date: - - Mon, 07 Jun 2021 09:14:51 GMT + - Mon, 24 Feb 2025 09:48:46 GMT server: - Kestrel Microsoft-HTTPAPI/2.0 strict-transport-security: - max-age=31536000; includeSubDomains - transfer-encoding: - - chunked x-ms-activity-id: - - 80b38c2c-9c27-4dcd-99c5-f21289990e1a + - 1bb9842a-21a4-4a7b-b91b-0f72ee3abd4e x-ms-job-clusterrequested-on: - 1/1/0001 12:00:00 AM +00:00 x-ms-job-ended-on: - - 6/7/2021 9:13:52 AM +00:00 + - 2/24/2025 9:47:49 AM +00:00 x-ms-job-internal-id: - - '423' + - '0' x-ms-job-livysubmission-on: - 1/1/0001 12:00:00 AM +00:00 x-ms-job-queued-on: @@ -394,13 +228,11 @@ interactions: x-ms-job-scheduler-state: - Ended x-ms-job-submitted-by-name: - - aaa@foo.com + - v-weishuzhao@microsoft.com x-ms-job-submitted-on: - - 6/7/2021 9:13:40 AM +00:00 + - 2/24/2025 9:47:41 AM +00:00 x-ms-job-type: - SparkServiceBatch - x-ms-response-time-ms: - - '26' status: code: 200 message: OK diff --git a/src/azure-cli/azure/cli/command_modules/synapse/tests/latest/recordings/test_spark_session_and_statements.yaml b/src/azure-cli/azure/cli/command_modules/synapse/tests/latest/recordings/test_spark_session_and_statements.yaml index e294643cd28..bd6390ec45d 100644 --- a/src/azure-cli/azure/cli/command_modules/synapse/tests/latest/recordings/test_spark_session_and_statements.yaml +++ b/src/azure-cli/azure/cli/command_modules/synapse/tests/latest/recordings/test_spark_session_and_statements.yaml @@ -10,35 +10,35 @@ interactions: Connection: - keep-alive Content-Length: - - '131' + - '133' Content-Type: - application/json User-Agent: - - azsdk-python-synapse/0.2.0 Python/3.7.8 (Windows-10-10.0.18362-SP0) + - azsdk-python-synapse-spark/0.7.0 Python/3.10.11 (Windows-10-10.0.26100-SP0) method: POST - uri: https://testsynapseworkspace.dev.azuresynapse.net/livyApi/versions/2019-11-01-preview/sparkPools/testsparkpool/sessions?detailed=true + uri: https://yanjuntestws001.dev.azuresynapse.net/livyApi/versions/2019-11-01-preview/sparkPools/sparkpooltest/sessions?detailed=true response: body: - string: '{"livyInfo":{"notStartedAt":null,"startingAt":null,"idleAt":null,"deadAt":null,"shuttingDownAt":null,"killedAt":null,"recoveringAt":null,"busyAt":null,"errorAt":null,"currentState":"not_started","jobCreationRequest":{"name":"clisession000002","conf":{},"driverMemory":"28g","driverCores":4,"executorMemory":"28g","executorCores":4,"numExecutors":2}},"name":"clisession000002","workspaceName":"testsynapseworkspace","computeName":"testsparkpool","sparkPoolName":"testsparkpool","submitterName":"aaa@foo.com","submitterId":"6ae2ff4b-4939-4952-a097-d234aad383a3","artifactId":"Livy","jobType":"SparkSession","result":"Uncertain","schedulerInfo":{"submittedAt":"2020-07-23T04:25:33.75932+00:00","scheduledAt":null,"endedAt":null,"cancellationRequestedAt":null,"currentState":"Queued"},"pluginInfo":{"preparationStartedAt":null,"resourceAcquisitionStartedAt":null,"submissionStartedAt":null,"monitoringStartedAt":null,"cleanupStartedAt":null,"currentState":"Preparation"},"tags":{},"id":10,"appId":null,"appInfo":null,"state":"not_started","log":null}' + string: '{"livyInfo":{"notStartedAt":null,"startingAt":null,"idleAt":null,"deadAt":null,"shuttingDownAt":null,"killedAt":null,"recoveringAt":null,"busyAt":null,"errorAt":null,"currentState":"not_started","jobCreationRequest":{"name":"clisession000002","conf":{},"driverMemory":"28g","driverCores":4,"executorMemory":"28g","executorCores":4,"numExecutors":2}},"name":"clisession000002","workspaceName":"yanjuntestws001","computeName":"sparkpooltest","sparkPoolName":"sparkpooltest","submitterName":"v-weishuzhao@microsoft.com","submitterId":"6a33200f-2b27-4fda-8567-b0c70d532441","artifactId":"Livy","jobType":"SparkSession","result":"Uncertain","schedulerInfo":{"submittedAt":"2025-02-24T10:41:45.6711287+00:00","queuedAt":"2025-02-24T10:41:45.8856562+00:00","scheduledAt":null,"endedAt":null,"cancellationRequestedAt":null,"currentState":"Queued"},"pluginInfo":{"preparationStartedAt":null,"resourceAcquisitionStartedAt":null,"submissionStartedAt":null,"monitoringStartedAt":null,"cleanupStartedAt":null,"currentState":"Preparation"},"tags":{},"id":4,"appId":null,"appInfo":null,"state":"not_started","log":null,"registeredSources":null}' headers: + content-length: + - '1129' content-type: - application/json; charset=utf-8 date: - - Thu, 23 Jul 2020 04:25:35 GMT + - Mon, 24 Feb 2025 10:41:45 GMT server: - Kestrel Microsoft-HTTPAPI/2.0 strict-transport-security: - max-age=31536000; includeSubDomains - transfer-encoding: - - chunked x-ms-activity-id: - - 493ed665-25c3-4f82-80e8-2bfd6079f761 + - 4e8e085b-c7e8-42c2-9f86-e60e05bb7659 x-ms-job-clusterrequested-on: - 1/1/0001 12:00:00 AM +00:00 x-ms-job-ended-on: - '' x-ms-job-internal-id: - - '10' + - '4' x-ms-job-livysubmission-on: - 1/1/0001 12:00:00 AM +00:00 x-ms-job-queued-on: @@ -50,13 +50,11 @@ interactions: x-ms-job-scheduler-state: - Queued x-ms-job-submitted-by-name: - - aaa@foo.com + - v-weishuzhao@microsoft.com x-ms-job-submitted-on: - - 7/23/2020 4:25:33 AM +00:00 + - 2/24/2025 10:41:45 AM +00:00 x-ms-job-type: - SparkServiceSession - x-ms-response-time-ms: - - '1399' status: code: 200 message: OK @@ -70,60 +68,51 @@ interactions: Connection: - keep-alive User-Agent: - - azsdk-python-synapse/0.2.0 Python/3.7.8 (Windows-10-10.0.18362-SP0) + - azsdk-python-synapse-spark/0.7.0 Python/3.10.11 (Windows-10-10.0.26100-SP0) method: GET - uri: https://testsynapseworkspace.dev.azuresynapse.net/livyApi/versions/2019-11-01-preview/sparkPools/testsparkpool/sessions/10?detailed=true + uri: https://yanjuntestws001.dev.azuresynapse.net/livyApi/versions/2019-11-01-preview/sparkPools/sparkpooltest/sessions/4?detailed=true response: body: - string: '{"livyInfo":{"notStartedAt":null,"startingAt":"2020-07-23T04:25:38.8804089+00:00","idleAt":"2020-07-23T04:26:04.0740324+00:00","deadAt":null,"shuttingDownAt":null,"killedAt":null,"recoveringAt":null,"busyAt":null,"errorAt":null,"currentState":"idle","jobCreationRequest":{"name":"clisession000002","conf":{},"driverMemory":"28g","driverCores":4,"executorMemory":"28g","executorCores":4,"numExecutors":2}},"name":"clisession000002","workspaceName":"testsynapseworkspace","computeName":"testsparkpool","sparkPoolName":"testsparkpool","submitterName":"aaa@foo.com","submitterId":"6ae2ff4b-4939-4952-a097-d234aad383a3","artifactId":"Livy","jobType":"SparkSession","result":"Uncertain","schedulerInfo":{"submittedAt":"2020-07-23T04:25:33.75932+00:00","scheduledAt":"2020-07-23T04:25:35.1475929+00:00","endedAt":null,"cancellationRequestedAt":null,"currentState":"Scheduled"},"pluginInfo":{"preparationStartedAt":"2020-07-23T04:25:35.1590862+00:00","resourceAcquisitionStartedAt":"2020-07-23T04:25:35.5201647+00:00","submissionStartedAt":"2020-07-23T04:25:38.6883539+00:00","monitoringStartedAt":"2020-07-23T04:25:38.8953971+00:00","cleanupStartedAt":null,"currentState":"Monitoring"},"tags":{},"id":10,"appId":"application_1595477936491_0002","appInfo":{"driverLogUrl":"http://da5952899ae247bbae2901a023b91b390006d921831:8042/node/containerlogs/container_1595477936491_0002_01_000001/trusted-service-user","sparkUiUrl":"http://da5952899ae247bbae2901a023b91b39003ef203634:8088/proxy/application_1595477936491_0002/"},"state":"idle","log":["\t - final status: UNDEFINED","\t tracking URL: http://da5952899ae247bbae2901a023b91b39003ef203634:8088/proxy/application_1595477936491_0002/","\t - user: trusted-service-user","20/07/23 04:25:43 INFO ShutdownHookManager: Shutdown - hook called","20/07/23 04:25:43 INFO ShutdownHookManager: Deleting directory - /tmp/spark-a26fc725-026a-4e91-9b8a-cb1994e2e6c8","20/07/23 04:25:43 INFO ShutdownHookManager: - Deleting directory /tmp/spark-0814d835-2f5f-45e5-a3cd-2d4a4695659d","20/07/23 - 04:25:43 INFO MetricsSystemImpl: Stopping azure-file-system metrics system...","20/07/23 - 04:25:43 INFO MetricsSystemImpl: azure-file-system metrics system stopped.","20/07/23 - 04:25:43 INFO MetricsSystemImpl: azure-file-system metrics system shutdown - complete.","\nYARN Diagnostics: "]}' + string: '{"livyInfo":{"notStartedAt":null,"startingAt":"2025-02-24T10:42:09.7963442+00:00","idleAt":"2025-02-24T10:42:41.9735224+00:00","deadAt":null,"shuttingDownAt":null,"killedAt":null,"recoveringAt":null,"busyAt":null,"errorAt":null,"currentState":"idle","jobCreationRequest":{"name":"clisession000002","conf":{},"driverMemory":"28g","driverCores":4,"executorMemory":"28g","executorCores":4,"numExecutors":2}},"name":"clisession000002","workspaceName":"yanjuntestws001","computeName":"sparkpooltest","sparkPoolName":"sparkpooltest","submitterName":"v-weishuzhao@microsoft.com","submitterId":"6a33200f-2b27-4fda-8567-b0c70d532441","artifactId":"Livy","jobType":"SparkSession","result":"Uncertain","schedulerInfo":{"submittedAt":"2025-02-24T10:41:45.6711287+00:00","queuedAt":"2025-02-24T10:41:45.8856562+00:00","scheduledAt":"2025-02-24T10:42:01.4498111+00:00","endedAt":null,"cancellationRequestedAt":null,"currentState":"Scheduled"},"pluginInfo":{"preparationStartedAt":"2025-02-24T10:42:03.460577+00:00","resourceAcquisitionStartedAt":"2025-02-24T10:42:05.4915055+00:00","submissionStartedAt":"2025-02-24T10:42:09.6944824+00:00","monitoringStartedAt":"2025-02-24T10:42:11.8224021+00:00","cleanupStartedAt":null,"currentState":"Monitoring"},"tags":{},"id":4,"appId":"application_1740390690825_0004","appInfo":{"sparkUiUrl":"https://web.azuresynapse.net/sparkui/72f988bf-86f1-41af-91ab-2d7cd011db47/workspaces/yanjuntestws001/sparkpools/sparkpooltest/sessions/4/applications/application_1740390690825_0004","impulseErrorCode":null,"isStreamingQueryExists":null,"impulseClassification":null,"impulseTsg":null,"isSessionTimedOut":"false","driverLogUrl":"https://web.azuresynapse.net/sparkui/72f988bf-86f1-41af-91ab-2d7cd011db47/api/v1/workspaces/yanjuntestws001/sparkpools/sparkpooltest/sessions/4/applications/application_1740390690825_0004/driverlog/stderr"},"state":"idle","log":["stdout: + ","Picked up _JAVA_OPTIONS: -Djava.io.tmpdir=/mnt/tmp","Picked up _JAVA_OPTIONS: + -Djava.io.tmpdir=/mnt/tmp","\nstderr: ","\nYARN Diagnostics: "],"registeredSources":["default"]}' headers: + content-length: + - '2062' content-type: - application/json; charset=utf-8 date: - - Thu, 23 Jul 2020 04:31:36 GMT + - Mon, 24 Feb 2025 10:47:46 GMT server: - Kestrel Microsoft-HTTPAPI/2.0 strict-transport-security: - max-age=31536000; includeSubDomains - transfer-encoding: - - chunked x-ms-activity-id: - - 93f0b804-3eb2-458e-b7f8-e3b2548682f8 + - ed31f2e0-1c46-4b2f-b346-27c654c2f94b x-ms-job-cluster: - - https://vnet-efcd4013-eastus-3018-1.eastus.cloudapp.azure.com:58977/ + - https://hubservice2.eastus.azuresynapse.net:8001/api/v1.0/publish/b52e52ce-76a1-4b6a-a1e8-4427905f745f x-ms-job-clusterrequested-on: - - 7/23/2020 4:25:35 AM +00:00 + - 1/1/0001 12:00:00 AM +00:00 x-ms-job-ended-on: - '' x-ms-job-internal-id: - - '10' + - '4' x-ms-job-livysubmission-on: - - 7/23/2020 4:25:38 AM +00:00 + - 2/24/2025 10:42:09 AM +00:00 x-ms-job-queued-on: - - 7/23/2020 4:25:38 AM +00:00 + - 2/24/2025 10:42:07 AM +00:00 x-ms-job-result: - Uncertain x-ms-job-scheduled-on: - - 7/23/2020 4:25:35 AM +00:00 + - 2/24/2025 10:42:01 AM +00:00 x-ms-job-scheduler-state: - Scheduled x-ms-job-submitted-by-name: - - aaa@foo.com + - v-weishuzhao@microsoft.com x-ms-job-submitted-on: - - 7/23/2020 4:25:33 AM +00:00 + - 2/24/2025 10:41:45 AM +00:00 x-ms-job-type: - SparkServiceSession - x-ms-response-time-ms: - - '19' status: code: 200 message: OK @@ -137,133 +126,35 @@ interactions: Connection: - keep-alive User-Agent: - - azsdk-python-synapse/0.2.0 Python/3.7.8 (Windows-10-10.0.18362-SP0) + - azsdk-python-synapse-spark/0.7.0 Python/3.10.11 (Windows-10-10.0.26100-SP0) method: GET - uri: https://testsynapseworkspace.dev.azuresynapse.net/livyApi/versions/2019-11-01-preview/sparkPools/testsparkpool/sessions?detailed=true + uri: https://yanjuntestws001.dev.azuresynapse.net/livyApi/versions/2019-11-01-preview/sparkPools/sparkpooltest/sessions?detailed=true response: body: - string: '{"from":0,"total":10,"sessions":[{"livyInfo":{"notStartedAt":null,"startingAt":"2020-07-22T11:31:34.7177616+00:00","idleAt":"2020-07-22T11:32:17.2894707+00:00","deadAt":"2020-07-22T11:55:09.9616988+00:00","shuttingDownAt":"2020-07-22T11:55:08.9255531+00:00","killedAt":null,"recoveringAt":null,"busyAt":"2020-07-22T11:43:54.1583695+00:00","errorAt":null,"currentState":"dead","jobCreationRequest":{"name":"clisessiontjrb","conf":{},"driverMemory":"28g","driverCores":4,"executorMemory":"28g","executorCores":4,"numExecutors":2}},"name":"clisessiontjrb","workspaceName":"testsynapseworkspace","computeName":"testsparkpool","sparkPoolName":"testsparkpool","submitterName":"aaa@foo.com","submitterId":"6ae2ff4b-4939-4952-a097-d234aad383a3","artifactId":"Livy","jobType":"SparkSession","result":"Failed","schedulerInfo":{"submittedAt":"2020-07-22T11:31:29.3129203+00:00","scheduledAt":"2020-07-22T11:31:30.4860661+00:00","endedAt":"2020-07-22T11:55:10.9479832+00:00","cancellationRequestedAt":null,"currentState":"Ended"},"pluginInfo":{"preparationStartedAt":"2020-07-22T11:31:30.4973199+00:00","resourceAcquisitionStartedAt":"2020-07-22T11:31:30.7527959+00:00","submissionStartedAt":"2020-07-22T11:31:33.8804769+00:00","monitoringStartedAt":"2020-07-22T11:31:34.7346934+00:00","cleanupStartedAt":"2020-07-22T11:55:09.9808091+00:00","currentState":"Ended"},"errorInfo":[{"message":"[plugins.testsynapseworkspace.testsparkpool.1] - [Monitoring] Livy Endpoint=[https://vnet-13a559a7-eastus-1015-2.eastus.cloudapp.azure.com:59340/]. - Livy Id=[0] Job failed during run time with state=[dead].","errorCode":"LIVY_JOB_STATE_DEAD","source":"Dependency"}],"tags":{},"id":1,"appId":"application_1595417168325_0001","appInfo":{"driverLogUrl":"http://91c1471bd1874780aa3cea29c7c9312a00245869534:8042/node/containerlogs/container_1595417168325_0001_01_000001/trusted-service-user","sparkUiUrl":"http://91c1471bd1874780aa3cea29c7c9312a00318090703:8088/proxy/application_1595417168325_0001/"},"state":"dead","log":["\t - final status: UNDEFINED","\t tracking URL: http://91c1471bd1874780aa3cea29c7c9312a00318090703:8088/proxy/application_1595417168325_0001/","\t - user: trusted-service-user","20/07/22 11:31:43 INFO ShutdownHookManager: Shutdown - hook called","20/07/22 11:31:43 INFO ShutdownHookManager: Deleting directory - /tmp/spark-d8f6b8fd-746b-46e1-acad-ddce3a1bdf53","20/07/22 11:31:44 INFO ShutdownHookManager: - Deleting directory /tmp/spark-8e4c5d19-49c9-4be9-9465-e35bf0d724e5","20/07/22 - 11:31:44 INFO MetricsSystemImpl: Stopping azure-file-system metrics system...","20/07/22 - 11:31:44 INFO MetricsSystemImpl: azure-file-system metrics system stopped.","20/07/22 - 11:31:44 INFO MetricsSystemImpl: azure-file-system metrics system shutdown - complete.","\nYARN Diagnostics: "]},{"livyInfo":{"notStartedAt":null,"startingAt":"2020-07-22T12:01:33.1288268+00:00","idleAt":"2020-07-22T12:02:10.4966838+00:00","deadAt":"2020-07-22T12:14:09.8659509+00:00","shuttingDownAt":null,"killedAt":null,"recoveringAt":null,"busyAt":"2020-07-22T12:03:12.2558048+00:00","errorAt":null,"currentState":"dead","jobCreationRequest":{"name":"sparktype","conf":{},"driverMemory":"28g","driverCores":4,"executorMemory":"28g","executorCores":4,"numExecutors":2}},"name":"sparktype","workspaceName":"testsynapseworkspace","computeName":"testsparkpool","sparkPoolName":"testsparkpool","submitterName":"aaa@foo.com","submitterId":"6ae2ff4b-4939-4952-a097-d234aad383a3","artifactId":"Livy","jobType":"SparkSession","result":"Failed","schedulerInfo":{"submittedAt":"2020-07-22T12:01:26.7367577+00:00","scheduledAt":"2020-07-22T12:01:28.9468706+00:00","endedAt":"2020-07-22T12:14:10.9475854+00:00","cancellationRequestedAt":null,"currentState":"Ended"},"pluginInfo":{"preparationStartedAt":"2020-07-22T12:01:28.9740669+00:00","resourceAcquisitionStartedAt":"2020-07-22T12:01:29.3395539+00:00","submissionStartedAt":"2020-07-22T12:01:33.0237529+00:00","monitoringStartedAt":"2020-07-22T12:01:33.1442534+00:00","cleanupStartedAt":"2020-07-22T12:14:09.8818771+00:00","currentState":"Ended"},"errorInfo":[{"message":"[plugins.testsynapseworkspace.testsparkpool.2] - [Monitoring] Livy Endpoint=[https://vnet-13a559a7-eastus-1015-2.eastus.cloudapp.azure.com:59340/]. - Livy Id=[1] Job failed during run time with state=[dead].","errorCode":"LIVY_JOB_STATE_DEAD","source":"Dependency"}],"tags":{},"id":2,"appId":"application_1595417168325_0002","appInfo":{"driverLogUrl":"http://91c1471bd1874780aa3cea29c7c9312a00318090703:8042/node/containerlogs/container_1595417168325_0002_01_000001/trusted-service-user","sparkUiUrl":"http://91c1471bd1874780aa3cea29c7c9312a00318090703:8088/proxy/application_1595417168325_0002/"},"state":"dead","log":["\t - final status: UNDEFINED","\t tracking URL: http://91c1471bd1874780aa3cea29c7c9312a00318090703:8088/proxy/application_1595417168325_0002/","\t - user: trusted-service-user","20/07/22 12:01:38 INFO ShutdownHookManager: Shutdown - hook called","20/07/22 12:01:38 INFO ShutdownHookManager: Deleting directory - /tmp/spark-07daa81a-ce9b-4db7-8515-04c46ac65f02","20/07/22 12:01:38 INFO ShutdownHookManager: - Deleting directory /tmp/spark-cc248879-0af9-4748-976f-470a919bea57","20/07/22 - 12:01:38 INFO MetricsSystemImpl: Stopping azure-file-system metrics system...","20/07/22 - 12:01:38 INFO MetricsSystemImpl: azure-file-system metrics system stopped.","20/07/22 - 12:01:38 INFO MetricsSystemImpl: azure-file-system metrics system shutdown - complete.","\nYARN Diagnostics: "]},{"livyInfo":{"notStartedAt":null,"startingAt":"2020-07-22T12:21:06.8635107+00:00","idleAt":"2020-07-22T12:21:27.0847891+00:00","deadAt":"2020-07-22T12:36:09.8009117+00:00","shuttingDownAt":null,"killedAt":null,"recoveringAt":null,"busyAt":null,"errorAt":null,"currentState":"dead","jobCreationRequest":{"name":"sparktype2","conf":{},"driverMemory":"28g","driverCores":4,"executorMemory":"28g","executorCores":4,"numExecutors":2}},"name":"sparktype2","workspaceName":"testsynapseworkspace","computeName":"testsparkpool","sparkPoolName":"testsparkpool","submitterName":"aaa@foo.com","submitterId":"6ae2ff4b-4939-4952-a097-d234aad383a3","artifactId":"Livy","jobType":"SparkSession","result":"Failed","schedulerInfo":{"submittedAt":"2020-07-22T12:21:02.0087644+00:00","scheduledAt":"2020-07-22T12:21:03.1109621+00:00","endedAt":"2020-07-22T12:36:10.8324212+00:00","cancellationRequestedAt":null,"currentState":"Ended"},"pluginInfo":{"preparationStartedAt":"2020-07-22T12:21:03.1241197+00:00","resourceAcquisitionStartedAt":"2020-07-22T12:21:03.3901832+00:00","submissionStartedAt":"2020-07-22T12:21:06.7629827+00:00","monitoringStartedAt":"2020-07-22T12:21:06.8781622+00:00","cleanupStartedAt":"2020-07-22T12:36:09.8421244+00:00","currentState":"Ended"},"errorInfo":[{"message":"[plugins.testsynapseworkspace.testsparkpool.3] - [Monitoring] Livy Endpoint=[https://vnet-13a559a7-eastus-1015-2.eastus.cloudapp.azure.com:59340/]. - Livy Id=[2] Job failed during run time with state=[dead].","errorCode":"LIVY_JOB_STATE_DEAD","source":"Dependency"}],"tags":{},"id":3,"appId":"application_1595417168325_0003","appInfo":{"driverLogUrl":"http://91c1471bd1874780aa3cea29c7c9312a00318090703:8042/node/containerlogs/container_1595417168325_0003_01_000001/trusted-service-user","sparkUiUrl":"http://91c1471bd1874780aa3cea29c7c9312a00318090703:8088/proxy/application_1595417168325_0003/"},"state":"dead","log":["\t - final status: UNDEFINED","\t tracking URL: http://91c1471bd1874780aa3cea29c7c9312a00318090703:8088/proxy/application_1595417168325_0003/","\t - user: trusted-service-user","20/07/22 12:21:12 INFO ShutdownHookManager: Shutdown - hook called","20/07/22 12:21:12 INFO ShutdownHookManager: Deleting directory - /tmp/spark-ea90cc04-05ae-4440-b5eb-b07a170587de","20/07/22 12:21:12 INFO ShutdownHookManager: - Deleting directory /tmp/spark-b69901f4-6793-4607-87c2-24abfb61b5b5","20/07/22 - 12:21:12 INFO MetricsSystemImpl: Stopping azure-file-system metrics system...","20/07/22 - 12:21:12 INFO MetricsSystemImpl: azure-file-system metrics system stopped.","20/07/22 - 12:21:12 INFO MetricsSystemImpl: azure-file-system metrics system shutdown - complete.","\nYARN Diagnostics: "]},{"livyInfo":{"notStartedAt":null,"startingAt":"2020-07-22T12:28:56.5869532+00:00","idleAt":"2020-07-22T12:29:21.8626678+00:00","deadAt":"2020-07-22T12:51:10.1998625+00:00","shuttingDownAt":"2020-07-22T12:51:09.1776045+00:00","killedAt":null,"recoveringAt":null,"busyAt":"2020-07-22T12:33:38.8281566+00:00","errorAt":null,"currentState":"dead","jobCreationRequest":{"name":"pyspark","conf":{},"driverMemory":"28g","driverCores":4,"executorMemory":"28g","executorCores":4,"numExecutors":2}},"name":"pyspark","workspaceName":"testsynapseworkspace","computeName":"testsparkpool","sparkPoolName":"testsparkpool","submitterName":"aaa@foo.com","submitterId":"6ae2ff4b-4939-4952-a097-d234aad383a3","artifactId":"Livy","jobType":"SparkSession","result":"Failed","schedulerInfo":{"submittedAt":"2020-07-22T12:28:51.9124286+00:00","scheduledAt":"2020-07-22T12:28:52.9667436+00:00","endedAt":"2020-07-22T12:51:11.3722098+00:00","cancellationRequestedAt":null,"currentState":"Ended"},"pluginInfo":{"preparationStartedAt":"2020-07-22T12:28:52.9778004+00:00","resourceAcquisitionStartedAt":"2020-07-22T12:28:53.2097181+00:00","submissionStartedAt":"2020-07-22T12:28:56.5149256+00:00","monitoringStartedAt":"2020-07-22T12:28:56.6054922+00:00","cleanupStartedAt":"2020-07-22T12:51:10.2422256+00:00","currentState":"Ended"},"errorInfo":[{"message":"[plugins.testsynapseworkspace.testsparkpool.4] - [Monitoring] Livy Endpoint=[https://vnet-13a559a7-eastus-1015-2.eastus.cloudapp.azure.com:59340/]. - Livy Id=[3] Job failed during run time with state=[dead].","errorCode":"LIVY_JOB_STATE_DEAD","source":"Dependency"}],"tags":{},"id":4,"appId":"application_1595417168325_0004","appInfo":{"driverLogUrl":"http://91c1471bd1874780aa3cea29c7c9312a001e6372339:8042/node/containerlogs/container_1595417168325_0004_01_000001/trusted-service-user","sparkUiUrl":"http://91c1471bd1874780aa3cea29c7c9312a00318090703:8088/proxy/application_1595417168325_0004/"},"state":"dead","log":["\t - final status: UNDEFINED","\t tracking URL: http://91c1471bd1874780aa3cea29c7c9312a00318090703:8088/proxy/application_1595417168325_0004/","\t - user: trusted-service-user","20/07/22 12:29:01 INFO ShutdownHookManager: Shutdown - hook called","20/07/22 12:29:01 INFO ShutdownHookManager: Deleting directory - /tmp/spark-4e5b24b9-02fc-4af9-b5f3-1694b1bb8fef","20/07/22 12:29:01 INFO ShutdownHookManager: - Deleting directory /tmp/spark-88d4b550-3a83-4968-b7c0-1b5cd2969667","20/07/22 - 12:29:01 INFO MetricsSystemImpl: Stopping azure-file-system metrics system...","20/07/22 - 12:29:01 INFO MetricsSystemImpl: azure-file-system metrics system stopped.","20/07/22 - 12:29:01 INFO MetricsSystemImpl: azure-file-system metrics system shutdown - complete.","\nYARN Diagnostics: "]},{"livyInfo":{"notStartedAt":null,"startingAt":"2020-07-22T13:03:02.5789197+00:00","idleAt":"2020-07-22T13:03:24.8039693+00:00","deadAt":"2020-07-22T13:14:09.4928943+00:00","shuttingDownAt":null,"killedAt":null,"recoveringAt":null,"busyAt":null,"errorAt":null,"currentState":"dead","jobCreationRequest":{"name":"pysparkkk","conf":{},"driverMemory":"28g","driverCores":4,"executorMemory":"28g","executorCores":4,"numExecutors":2}},"name":"pysparkkk","workspaceName":"testsynapseworkspace","computeName":"testsparkpool","sparkPoolName":"testsparkpool","submitterName":"aaa@foo.com","submitterId":"6ae2ff4b-4939-4952-a097-d234aad383a3","artifactId":"Livy","jobType":"SparkSession","result":"Failed","schedulerInfo":{"submittedAt":"2020-07-22T13:02:56.8892002+00:00","scheduledAt":"2020-07-22T13:02:57.9809006+00:00","endedAt":"2020-07-22T13:14:10.6470835+00:00","cancellationRequestedAt":null,"currentState":"Ended"},"pluginInfo":{"preparationStartedAt":"2020-07-22T13:02:57.9929062+00:00","resourceAcquisitionStartedAt":"2020-07-22T13:02:58.3801697+00:00","submissionStartedAt":"2020-07-22T13:03:02.4719813+00:00","monitoringStartedAt":"2020-07-22T13:03:02.594057+00:00","cleanupStartedAt":"2020-07-22T13:14:09.5113401+00:00","currentState":"Ended"},"errorInfo":[{"message":"[plugins.testsynapseworkspace.testsparkpool.5] - [Monitoring] Livy Endpoint=[https://vnet-13a559a7-eastus-1015-2.eastus.cloudapp.azure.com:59340/]. - Livy Id=[4] Job failed during run time with state=[dead].","errorCode":"LIVY_JOB_STATE_DEAD","source":"Dependency"}],"tags":{},"id":5,"appId":"application_1595417168325_0005","appInfo":{"driverLogUrl":"http://91c1471bd1874780aa3cea29c7c9312a001e6372339:8042/node/containerlogs/container_1595417168325_0005_01_000001/trusted-service-user","sparkUiUrl":"http://91c1471bd1874780aa3cea29c7c9312a00318090703:8088/proxy/application_1595417168325_0005/"},"state":"dead","log":["\t - final status: UNDEFINED","\t tracking URL: http://91c1471bd1874780aa3cea29c7c9312a00318090703:8088/proxy/application_1595417168325_0005/","\t - user: trusted-service-user","20/07/22 13:03:07 INFO ShutdownHookManager: Shutdown - hook called","20/07/22 13:03:07 INFO ShutdownHookManager: Deleting directory - /tmp/spark-3ad35b49-4e4c-40d9-bcfb-f38b395bf2df","20/07/22 13:03:07 INFO ShutdownHookManager: - Deleting directory /tmp/spark-2305624a-ddae-4050-a459-c07903dae3fd","20/07/22 - 13:03:07 INFO MetricsSystemImpl: Stopping azure-file-system metrics system...","20/07/22 - 13:03:07 INFO MetricsSystemImpl: azure-file-system metrics system stopped.","20/07/22 - 13:03:07 INFO MetricsSystemImpl: azure-file-system metrics system shutdown - complete.","\nYARN Diagnostics: "]},{"livyInfo":{"notStartedAt":null,"startingAt":"2020-07-23T02:54:08.3015433+00:00","idleAt":"2020-07-23T02:54:52.9122213+00:00","deadAt":"2020-07-23T03:08:53.4796427+00:00","shuttingDownAt":"2020-07-23T03:08:49.4381399+00:00","killedAt":null,"recoveringAt":null,"busyAt":"2020-07-23T02:58:18.1446859+00:00","errorAt":null,"currentState":"dead","jobCreationRequest":{"name":"clisessionvqgf","conf":{},"driverMemory":"28g","driverCores":4,"executorMemory":"28g","executorCores":4,"numExecutors":2}},"name":"clisessionvqgf","workspaceName":"testsynapseworkspace","computeName":"testsparkpool","sparkPoolName":"testsparkpool","submitterName":"aaa@foo.com","submitterId":"6ae2ff4b-4939-4952-a097-d234aad383a3","artifactId":"Livy","jobType":"SparkSession","result":"Failed","schedulerInfo":{"submittedAt":"2020-07-23T02:52:09.5849732+00:00","scheduledAt":"2020-07-23T02:52:10.7972332+00:00","endedAt":"2020-07-23T03:08:54.9128812+00:00","cancellationRequestedAt":null,"currentState":"Ended"},"pluginInfo":{"preparationStartedAt":"2020-07-23T02:52:10.8098588+00:00","resourceAcquisitionStartedAt":"2020-07-23T02:52:11.1037299+00:00","submissionStartedAt":"2020-07-23T02:54:07.5147441+00:00","monitoringStartedAt":"2020-07-23T02:54:08.3233251+00:00","cleanupStartedAt":"2020-07-23T03:08:53.5705885+00:00","currentState":"Ended"},"errorInfo":[{"message":"[plugins.testsynapseworkspace.testsparkpool.6] - [Monitoring] Livy Endpoint=[https://vnet-b3ea47a2-eastus-1016-2.eastus.cloudapp.azure.com:58951/]. - Livy Id=[0] Job failed during run time with state=[dead].","errorCode":"LIVY_JOB_STATE_DEAD","source":"Dependency"}],"tags":{},"id":6,"appId":"application_1595472828052_0001","appInfo":{"driverLogUrl":"http://a01a06d90545412f9479d89964bd7006000a8a00779:8042/node/containerlogs/container_1595472828052_0001_01_000001/trusted-service-user","sparkUiUrl":"http://a01a06d90545412f9479d89964bd7006000a8a00779:8088/proxy/application_1595472828052_0001/"},"state":"dead","log":["\t - final status: UNDEFINED","\t tracking URL: http://a01a06d90545412f9479d89964bd7006000a8a00779:8088/proxy/application_1595472828052_0001/","\t - user: trusted-service-user","20/07/23 02:54:15 INFO ShutdownHookManager: Shutdown - hook called","20/07/23 02:54:15 INFO ShutdownHookManager: Deleting directory - /tmp/spark-270e4cae-7489-458e-a3d1-1aca38dac89a","20/07/23 02:54:15 INFO ShutdownHookManager: - Deleting directory /tmp/spark-23369e60-1172-414a-994b-7264caef936e","20/07/23 - 02:54:15 INFO MetricsSystemImpl: Stopping azure-file-system metrics system...","20/07/23 - 02:54:15 INFO MetricsSystemImpl: azure-file-system metrics system stopped.","20/07/23 - 02:54:15 INFO MetricsSystemImpl: azure-file-system metrics system shutdown - complete.","\nYARN Diagnostics: "]},{"livyInfo":{"notStartedAt":null,"startingAt":"2020-07-23T03:04:29.1993908+00:00","idleAt":"2020-07-23T03:04:54.4640762+00:00","deadAt":"2020-07-23T03:20:48.9302008+00:00","shuttingDownAt":null,"killedAt":null,"recoveringAt":null,"busyAt":null,"errorAt":null,"currentState":"dead","jobCreationRequest":{"name":"clisessionf76a","conf":{},"driverMemory":"28g","driverCores":4,"executorMemory":"28g","executorCores":4,"numExecutors":2}},"name":"clisessionf76a","workspaceName":"testsynapseworkspace","computeName":"testsparkpool","sparkPoolName":"testsparkpool","submitterName":"aaa@foo.com","submitterId":"6ae2ff4b-4939-4952-a097-d234aad383a3","artifactId":"Livy","jobType":"SparkSession","result":"Failed","schedulerInfo":{"submittedAt":"2020-07-23T03:04:24.1106704+00:00","scheduledAt":"2020-07-23T03:04:25.3853695+00:00","endedAt":"2020-07-23T03:20:50.0570311+00:00","cancellationRequestedAt":null,"currentState":"Ended"},"pluginInfo":{"preparationStartedAt":"2020-07-23T03:04:25.3982316+00:00","resourceAcquisitionStartedAt":"2020-07-23T03:04:25.6467546+00:00","submissionStartedAt":"2020-07-23T03:04:29.0445766+00:00","monitoringStartedAt":"2020-07-23T03:04:29.2136644+00:00","cleanupStartedAt":"2020-07-23T03:20:48.9715223+00:00","currentState":"Ended"},"errorInfo":[{"message":"[plugins.testsynapseworkspace.testsparkpool.7] - [Monitoring] Livy Endpoint=[https://vnet-b3ea47a2-eastus-1016-2.eastus.cloudapp.azure.com:58951/]. - Livy Id=[1] Job failed during run time with state=[dead].","errorCode":"LIVY_JOB_STATE_DEAD","source":"Dependency"}],"tags":{},"id":7,"appId":"application_1595472828052_0002","appInfo":{"driverLogUrl":"http://a01a06d90545412f9479d89964bd7006003d7270852:8042/node/containerlogs/container_1595472828052_0002_01_000001/trusted-service-user","sparkUiUrl":"http://a01a06d90545412f9479d89964bd7006000a8a00779:8088/proxy/application_1595472828052_0002/"},"state":"dead","log":["\t - final status: UNDEFINED","\t tracking URL: http://a01a06d90545412f9479d89964bd7006000a8a00779:8088/proxy/application_1595472828052_0002/","\t - user: trusted-service-user","20/07/23 03:04:33 INFO ShutdownHookManager: Shutdown - hook called","20/07/23 03:04:33 INFO ShutdownHookManager: Deleting directory - /tmp/spark-b942770d-a4f4-4274-aab6-dd8d9b42d534","20/07/23 03:04:33 INFO ShutdownHookManager: - Deleting directory /tmp/spark-1e6e00dd-61cf-4c35-8eae-b323f9f2d346","20/07/23 - 03:04:33 INFO MetricsSystemImpl: Stopping azure-file-system metrics system...","20/07/23 - 03:04:33 INFO MetricsSystemImpl: azure-file-system metrics system stopped.","20/07/23 - 03:04:33 INFO MetricsSystemImpl: azure-file-system metrics system shutdown - complete.","\nYARN Diagnostics: "]},{"livyInfo":{"notStartedAt":null,"startingAt":"2020-07-23T03:18:33.4857432+00:00","idleAt":"2020-07-23T03:18:54.7167106+00:00","deadAt":"2020-07-23T03:34:49.8941237+00:00","shuttingDownAt":"2020-07-23T03:34:48.8733343+00:00","killedAt":null,"recoveringAt":null,"busyAt":null,"errorAt":null,"currentState":"dead","jobCreationRequest":{"name":"clisessionrqug","conf":{},"driverMemory":"28g","driverCores":4,"executorMemory":"28g","executorCores":4,"numExecutors":2}},"name":"clisessionrqug","workspaceName":"testsynapseworkspace","computeName":"testsparkpool","sparkPoolName":"testsparkpool","submitterName":"aaa@foo.com","submitterId":"6ae2ff4b-4939-4952-a097-d234aad383a3","artifactId":"Livy","jobType":"SparkSession","result":"Failed","schedulerInfo":{"submittedAt":"2020-07-23T03:18:28.1253582+00:00","scheduledAt":"2020-07-23T03:18:29.1593842+00:00","endedAt":"2020-07-23T03:34:52.5271744+00:00","cancellationRequestedAt":null,"currentState":"Ended"},"pluginInfo":{"preparationStartedAt":"2020-07-23T03:18:29.1709816+00:00","resourceAcquisitionStartedAt":"2020-07-23T03:18:29.5266364+00:00","submissionStartedAt":"2020-07-23T03:18:33.3926116+00:00","monitoringStartedAt":"2020-07-23T03:18:33.5014585+00:00","cleanupStartedAt":"2020-07-23T03:34:49.9355364+00:00","currentState":"Ended"},"errorInfo":[{"message":"[plugins.testsynapseworkspace.testsparkpool.8] - [Monitoring] Livy Endpoint=[https://vnet-b3ea47a2-eastus-1016-2.eastus.cloudapp.azure.com:58951/]. - Livy Id=[2] Job failed during run time with state=[dead].","errorCode":"LIVY_JOB_STATE_DEAD","source":"Dependency"}],"tags":{},"id":8,"appId":"application_1595472828052_0003","appInfo":{"driverLogUrl":"http://a01a06d90545412f9479d89964bd7006000a8a00779:8042/node/containerlogs/container_1595472828052_0003_01_000001/trusted-service-user","sparkUiUrl":"http://a01a06d90545412f9479d89964bd7006000a8a00779:8088/proxy/application_1595472828052_0003/"},"state":"dead","log":["\t - final status: UNDEFINED","\t tracking URL: http://a01a06d90545412f9479d89964bd7006000a8a00779:8088/proxy/application_1595472828052_0003/","\t - user: trusted-service-user","20/07/23 03:18:38 INFO ShutdownHookManager: Shutdown - hook called","20/07/23 03:18:38 INFO ShutdownHookManager: Deleting directory - /tmp/spark-ef1f99f1-3b57-4d4f-8302-802cd0d66284","20/07/23 03:18:38 INFO ShutdownHookManager: - Deleting directory /tmp/spark-8c09f552-a2f4-4a53-9c2a-e59614d4bd64","20/07/23 - 03:18:38 INFO MetricsSystemImpl: Stopping azure-file-system metrics system...","20/07/23 - 03:18:38 INFO MetricsSystemImpl: azure-file-system metrics system stopped.","20/07/23 - 03:18:38 INFO MetricsSystemImpl: azure-file-system metrics system shutdown - complete.","\nYARN Diagnostics: "]},{"livyInfo":{"notStartedAt":null,"startingAt":"2020-07-23T04:19:18.5571383+00:00","idleAt":"2020-07-23T04:20:05.0867435+00:00","deadAt":null,"shuttingDownAt":null,"killedAt":null,"recoveringAt":null,"busyAt":"2020-07-23T04:23:27.4944228+00:00","errorAt":null,"currentState":"idle","jobCreationRequest":{"name":"clisession3ozp","conf":{},"driverMemory":"28g","driverCores":4,"executorMemory":"28g","executorCores":4,"numExecutors":2}},"name":"clisession3ozp","workspaceName":"testsynapseworkspace","computeName":"testsparkpool","sparkPoolName":"testsparkpool","submitterName":"aaa@foo.com","submitterId":"6ae2ff4b-4939-4952-a097-d234aad383a3","artifactId":"Livy","jobType":"SparkSession","result":"Uncertain","schedulerInfo":{"submittedAt":"2020-07-23T04:17:17.6466905+00:00","scheduledAt":"2020-07-23T04:17:18.6916675+00:00","endedAt":null,"cancellationRequestedAt":null,"currentState":"Scheduled"},"pluginInfo":{"preparationStartedAt":"2020-07-23T04:17:18.7075036+00:00","resourceAcquisitionStartedAt":"2020-07-23T04:17:18.9396798+00:00","submissionStartedAt":"2020-07-23T04:19:17.5901861+00:00","monitoringStartedAt":"2020-07-23T04:19:18.5719766+00:00","cleanupStartedAt":null,"currentState":"Monitoring"},"tags":{},"id":9,"appId":"application_1595477936491_0001","appInfo":{"driverLogUrl":"http://da5952899ae247bbae2901a023b91b39003ef203634:8042/node/containerlogs/container_1595477936491_0001_01_000001/trusted-service-user","sparkUiUrl":"http://da5952899ae247bbae2901a023b91b39003ef203634:8088/proxy/application_1595477936491_0001/"},"state":"idle","log":["\t - final status: UNDEFINED","\t tracking URL: http://da5952899ae247bbae2901a023b91b39003ef203634:8088/proxy/application_1595477936491_0001/","\t - user: trusted-service-user","20/07/23 04:19:26 INFO ShutdownHookManager: Shutdown - hook called","20/07/23 04:19:26 INFO ShutdownHookManager: Deleting directory - /tmp/spark-16e620c8-c0a6-40ee-9f4f-459f3b519e32","20/07/23 04:19:26 INFO ShutdownHookManager: - Deleting directory /tmp/spark-9b18306d-a717-433b-8d44-252fcd25bd33","20/07/23 - 04:19:26 INFO MetricsSystemImpl: Stopping azure-file-system metrics system...","20/07/23 - 04:19:26 INFO MetricsSystemImpl: azure-file-system metrics system stopped.","20/07/23 - 04:19:26 INFO MetricsSystemImpl: azure-file-system metrics system shutdown - complete.","\nYARN Diagnostics: "]},{"livyInfo":{"notStartedAt":null,"startingAt":"2020-07-23T04:25:38.8804089+00:00","idleAt":"2020-07-23T04:26:04.0740324+00:00","deadAt":null,"shuttingDownAt":null,"killedAt":null,"recoveringAt":null,"busyAt":null,"errorAt":null,"currentState":"idle","jobCreationRequest":{"name":"clisession000002","conf":{},"driverMemory":"28g","driverCores":4,"executorMemory":"28g","executorCores":4,"numExecutors":2}},"name":"clisession000002","workspaceName":"testsynapseworkspace","computeName":"testsparkpool","sparkPoolName":"testsparkpool","submitterName":"aaa@foo.com","submitterId":"6ae2ff4b-4939-4952-a097-d234aad383a3","artifactId":"Livy","jobType":"SparkSession","result":"Uncertain","schedulerInfo":{"submittedAt":"2020-07-23T04:25:33.75932+00:00","scheduledAt":"2020-07-23T04:25:35.1475929+00:00","endedAt":null,"cancellationRequestedAt":null,"currentState":"Scheduled"},"pluginInfo":{"preparationStartedAt":"2020-07-23T04:25:35.1590862+00:00","resourceAcquisitionStartedAt":"2020-07-23T04:25:35.5201647+00:00","submissionStartedAt":"2020-07-23T04:25:38.6883539+00:00","monitoringStartedAt":"2020-07-23T04:25:38.8953971+00:00","cleanupStartedAt":null,"currentState":"Monitoring"},"tags":{},"id":10,"appId":"application_1595477936491_0002","appInfo":{"driverLogUrl":"http://da5952899ae247bbae2901a023b91b390006d921831:8042/node/containerlogs/container_1595477936491_0002_01_000001/trusted-service-user","sparkUiUrl":"http://da5952899ae247bbae2901a023b91b39003ef203634:8088/proxy/application_1595477936491_0002/"},"state":"idle","log":["\t - final status: UNDEFINED","\t tracking URL: http://da5952899ae247bbae2901a023b91b39003ef203634:8088/proxy/application_1595477936491_0002/","\t - user: trusted-service-user","20/07/23 04:25:43 INFO ShutdownHookManager: Shutdown - hook called","20/07/23 04:25:43 INFO ShutdownHookManager: Deleting directory - /tmp/spark-a26fc725-026a-4e91-9b8a-cb1994e2e6c8","20/07/23 04:25:43 INFO ShutdownHookManager: - Deleting directory /tmp/spark-0814d835-2f5f-45e5-a3cd-2d4a4695659d","20/07/23 - 04:25:43 INFO MetricsSystemImpl: Stopping azure-file-system metrics system...","20/07/23 - 04:25:43 INFO MetricsSystemImpl: azure-file-system metrics system stopped.","20/07/23 - 04:25:43 INFO MetricsSystemImpl: azure-file-system metrics system shutdown - complete.","\nYARN Diagnostics: "]}]}' + string: '{"from":0,"total":4,"sessions":[{"livyInfo":{"notStartedAt":null,"startingAt":"2025-02-24T10:42:09.7963442+00:00","idleAt":"2025-02-24T10:42:41.9735224+00:00","deadAt":null,"shuttingDownAt":null,"killedAt":null,"recoveringAt":null,"busyAt":null,"errorAt":null,"currentState":"idle","jobCreationRequest":{"name":"clisession000002","conf":{},"driverMemory":"28g","driverCores":4,"executorMemory":"28g","executorCores":4,"numExecutors":2}},"name":"clisession000002","workspaceName":"yanjuntestws001","computeName":"sparkpooltest","sparkPoolName":"sparkpooltest","submitterName":"v-weishuzhao@microsoft.com","submitterId":"6a33200f-2b27-4fda-8567-b0c70d532441","artifactId":"Livy","jobType":"SparkSession","result":"Uncertain","schedulerInfo":{"submittedAt":"2025-02-24T10:41:45.6711287+00:00","queuedAt":"2025-02-24T10:41:45.8856562+00:00","scheduledAt":"2025-02-24T10:42:01.4498111+00:00","endedAt":null,"cancellationRequestedAt":null,"currentState":"Scheduled"},"pluginInfo":{"preparationStartedAt":"2025-02-24T10:42:03.460577+00:00","resourceAcquisitionStartedAt":"2025-02-24T10:42:05.4915055+00:00","submissionStartedAt":"2025-02-24T10:42:09.6944824+00:00","monitoringStartedAt":"2025-02-24T10:42:11.8224021+00:00","cleanupStartedAt":null,"currentState":"Monitoring"},"tags":{},"id":4,"appId":"application_1740390690825_0004","appInfo":{"sparkUiUrl":"https://web.azuresynapse.net/sparkui/72f988bf-86f1-41af-91ab-2d7cd011db47/workspaces/yanjuntestws001/sparkpools/sparkpooltest/sessions/4/applications/application_1740390690825_0004","impulseErrorCode":null,"isStreamingQueryExists":null,"impulseClassification":null,"impulseTsg":null,"isSessionTimedOut":"false","driverLogUrl":"https://web.azuresynapse.net/sparkui/72f988bf-86f1-41af-91ab-2d7cd011db47/api/v1/workspaces/yanjuntestws001/sparkpools/sparkpooltest/sessions/4/applications/application_1740390690825_0004/driverlog/stderr"},"state":"idle","log":["stdout: + ","Picked up _JAVA_OPTIONS: -Djava.io.tmpdir=/mnt/tmp","Picked up _JAVA_OPTIONS: + -Djava.io.tmpdir=/mnt/tmp","\nstderr: ","\nYARN Diagnostics: "],"registeredSources":["default"]},{"livyInfo":{"notStartedAt":null,"startingAt":"2025-02-24T10:32:17.9942634+00:00","idleAt":"2025-02-24T10:32:50.1367932+00:00","deadAt":null,"shuttingDownAt":null,"killedAt":null,"recoveringAt":null,"busyAt":"2025-02-24T10:38:03.7296969+00:00","errorAt":null,"currentState":"idle","jobCreationRequest":{"name":"clisessionwlko","conf":{},"driverMemory":"28g","driverCores":4,"executorMemory":"28g","executorCores":4,"numExecutors":2}},"name":"clisessionwlko","workspaceName":"yanjuntestws001","computeName":"sparkpooltest","sparkPoolName":"sparkpooltest","submitterName":"v-weishuzhao@microsoft.com","submitterId":"6a33200f-2b27-4fda-8567-b0c70d532441","artifactId":"Livy","jobType":"SparkSession","result":"Uncertain","schedulerInfo":{"submittedAt":"2025-02-24T10:31:56.173809+00:00","queuedAt":"2025-02-24T10:31:56.4140488+00:00","scheduledAt":"2025-02-24T10:32:09.429065+00:00","endedAt":null,"cancellationRequestedAt":null,"currentState":"Scheduled"},"pluginInfo":{"preparationStartedAt":"2025-02-24T10:32:11.4528594+00:00","resourceAcquisitionStartedAt":"2025-02-24T10:32:13.4810608+00:00","submissionStartedAt":"2025-02-24T10:32:17.6692179+00:00","monitoringStartedAt":"2025-02-24T10:32:20.0121583+00:00","cleanupStartedAt":null,"currentState":"Monitoring"},"tags":{},"id":3,"appId":"application_1740390690825_0003","appInfo":{"sparkUiUrl":"https://web.azuresynapse.net/sparkui/72f988bf-86f1-41af-91ab-2d7cd011db47/workspaces/yanjuntestws001/sparkpools/sparkpooltest/sessions/3/applications/application_1740390690825_0003","impulseErrorCode":null,"isStreamingQueryExists":"false","impulseClassification":null,"impulseTsg":null,"isSessionTimedOut":"false","driverLogUrl":"https://web.azuresynapse.net/sparkui/72f988bf-86f1-41af-91ab-2d7cd011db47/api/v1/workspaces/yanjuntestws001/sparkpools/sparkpooltest/sessions/3/applications/application_1740390690825_0003/driverlog/stderr"},"state":"idle","log":["stdout: + ","Picked up _JAVA_OPTIONS: -Djava.io.tmpdir=/mnt/tmp","Picked up _JAVA_OPTIONS: + -Djava.io.tmpdir=/mnt/tmp","\nstderr: ","\nYARN Diagnostics: "],"registeredSources":["default"]},{"livyInfo":{"notStartedAt":null,"startingAt":"2025-02-24T09:59:48.5698+00:00","idleAt":"2025-02-24T10:00:30.8327824+00:00","deadAt":"2025-02-24T10:26:27.8209956+00:00","shuttingDownAt":null,"killedAt":null,"recoveringAt":null,"busyAt":"2025-02-24T10:05:46.7830254+00:00","errorAt":null,"currentState":"dead","jobCreationRequest":{"name":"clisessions3ki","conf":{},"driverMemory":"28g","driverCores":4,"executorMemory":"28g","executorCores":4,"numExecutors":2}},"name":"clisessions3ki","workspaceName":"yanjuntestws001","computeName":"sparkpooltest","sparkPoolName":"sparkpooltest","submitterName":"v-weishuzhao@microsoft.com","submitterId":"6a33200f-2b27-4fda-8567-b0c70d532441","artifactId":"Livy","jobType":"SparkSession","result":"Failed","schedulerInfo":{"submittedAt":"2025-02-24T09:59:27.2472085+00:00","queuedAt":"2025-02-24T09:59:27.4628781+00:00","scheduledAt":"2025-02-24T09:59:40.1937144+00:00","endedAt":"2025-02-24T10:26:58.7684231+00:00","cancellationRequestedAt":null,"currentState":"Ended"},"pluginInfo":{"preparationStartedAt":"2025-02-24T09:59:42.2200781+00:00","resourceAcquisitionStartedAt":"2025-02-24T09:59:44.238812+00:00","submissionStartedAt":"2025-02-24T09:59:48.4389784+00:00","monitoringStartedAt":"2025-02-24T09:59:50.5955258+00:00","cleanupStartedAt":"2025-02-24T10:26:29.8524773+00:00","currentState":"Ended"},"errorInfo":[{"message":"Job + failed during run time with state=[dead].","errorCode":"LIVY_JOB_TIMED_OUT","source":"Unknown"}],"tags":{},"id":2,"appId":"application_1740390690825_0002","appInfo":{"sparkUiUrl":"https://web.azuresynapse.net/sparkui/72f988bf-86f1-41af-91ab-2d7cd011db47/workspaces/yanjuntestws001/sparkpools/sparkpooltest/sessions/2/applications/application_1740390690825_0002","impulseErrorCode":null,"isStreamingQueryExists":"false","impulseClassification":null,"impulseTsg":null,"isSessionTimedOut":"true","driverLogUrl":"https://web.azuresynapse.net/sparkui/72f988bf-86f1-41af-91ab-2d7cd011db47/api/v1/workspaces/yanjuntestws001/sparkpools/sparkpooltest/sessions/2/applications/application_1740390690825_0002/driverlog/stderr"},"state":"dead","log":["stdout: + ","Picked up _JAVA_OPTIONS: -Djava.io.tmpdir=/mnt/tmp","Picked up _JAVA_OPTIONS: + -Djava.io.tmpdir=/mnt/tmp","\nstderr: ","\nYARN Diagnostics: "],"registeredSources":["default"]},{"livyInfo":{"notStartedAt":null,"startingAt":"2025-02-24T09:51:33.2339694+00:00","idleAt":"2025-02-24T09:52:37.3843807+00:00","deadAt":"2025-02-24T10:16:33.5268861+00:00","shuttingDownAt":null,"killedAt":null,"recoveringAt":null,"busyAt":"2025-02-24T09:56:11.4409558+00:00","errorAt":null,"currentState":"dead","jobCreationRequest":{"name":"clisessionv4gv","conf":{},"driverMemory":"28g","driverCores":4,"executorMemory":"28g","executorCores":4,"numExecutors":2}},"name":"clisessionv4gv","workspaceName":"yanjuntestws001","computeName":"sparkpooltest","sparkPoolName":"sparkpooltest","submitterName":"v-weishuzhao@microsoft.com","submitterId":"6a33200f-2b27-4fda-8567-b0c70d532441","artifactId":"Livy","jobType":"SparkSession","result":"Failed","schedulerInfo":{"submittedAt":"2025-02-24T09:49:54.3910146+00:00","queuedAt":"2025-02-24T09:49:54.4377299+00:00","scheduledAt":"2025-02-24T09:50:07.9470742+00:00","endedAt":"2025-02-24T10:17:03.2997531+00:00","cancellationRequestedAt":null,"currentState":"Ended"},"pluginInfo":{"preparationStartedAt":"2025-02-24T09:50:09.960312+00:00","resourceAcquisitionStartedAt":"2025-02-24T09:50:11.9738955+00:00","submissionStartedAt":"2025-02-24T09:51:26.2698889+00:00","monitoringStartedAt":"2025-02-24T09:51:35.2575408+00:00","cleanupStartedAt":"2025-02-24T10:16:35.5386462+00:00","currentState":"Ended"},"errorInfo":[{"message":"Job + failed during run time with state=[dead].","errorCode":"LIVY_JOB_TIMED_OUT","source":"Unknown"}],"tags":{},"id":1,"appId":"application_1740390690825_0001","appInfo":{"sparkUiUrl":"https://web.azuresynapse.net/sparkui/72f988bf-86f1-41af-91ab-2d7cd011db47/workspaces/yanjuntestws001/sparkpools/sparkpooltest/sessions/1/applications/application_1740390690825_0001","impulseErrorCode":null,"isStreamingQueryExists":"false","impulseClassification":null,"impulseTsg":null,"isSessionTimedOut":"true","driverLogUrl":"https://web.azuresynapse.net/sparkui/72f988bf-86f1-41af-91ab-2d7cd011db47/api/v1/workspaces/yanjuntestws001/sparkpools/sparkpooltest/sessions/1/applications/application_1740390690825_0001/driverlog/stderr"},"state":"dead","log":["stdout: + ","Picked up _JAVA_OPTIONS: -Djava.io.tmpdir=/mnt/tmp","Picked up _JAVA_OPTIONS: + -Djava.io.tmpdir=/mnt/tmp","\nstderr: ","\nYARN Diagnostics: "],"registeredSources":["default"]}]}' headers: + content-length: + - '8783' content-type: - application/json; charset=utf-8 date: - - Thu, 23 Jul 2020 04:31:37 GMT + - Mon, 24 Feb 2025 10:47:47 GMT server: - Kestrel Microsoft-HTTPAPI/2.0 strict-transport-security: - max-age=31536000; includeSubDomains - transfer-encoding: - - chunked x-ms-activity-id: - - 5aa94afa-7738-4d46-b3e1-1db4de94be6a - x-ms-response-time-ms: - - '75' + - 5fc02b86-c436-4359-96be-dc4fa2181ea2 status: code: 200 message: OK @@ -279,9 +170,9 @@ interactions: Content-Length: - '0' User-Agent: - - azsdk-python-synapse/0.2.0 Python/3.7.8 (Windows-10-10.0.18362-SP0) + - azsdk-python-synapse-spark/0.7.0 Python/3.10.11 (Windows-10-10.0.26100-SP0) method: PUT - uri: https://testsynapseworkspace.dev.azuresynapse.net/livyApi/versions/2019-11-01-preview/sparkPools/testsparkpool/sessions/10/reset-timeout + uri: https://yanjuntestws001.dev.azuresynapse.net/livyApi/versions/2019-11-01-preview/sparkPools/sparkpooltest/sessions/4/reset-timeout response: body: string: '' @@ -289,15 +180,15 @@ interactions: content-length: - '0' date: - - Thu, 23 Jul 2020 04:31:39 GMT + - Mon, 24 Feb 2025 10:47:49 GMT server: - Kestrel Microsoft-HTTPAPI/2.0 strict-transport-security: - max-age=31536000; includeSubDomains x-ms-activity-id: - - 6d05a89d-9c76-4dd5-9039-d90d1d696529 + - 5565a4ab-6736-4de0-8871-ad9a4c486ca4 x-ms-response-time-ms: - - '55' + - '50' status: code: 200 message: OK @@ -316,27 +207,25 @@ interactions: Content-Type: - application/json User-Agent: - - azsdk-python-synapse/0.2.0 Python/3.7.8 (Windows-10-10.0.18362-SP0) + - azsdk-python-synapse-spark/0.7.0 Python/3.10.11 (Windows-10-10.0.26100-SP0) method: POST - uri: https://testsynapseworkspace.dev.azuresynapse.net/livyApi/versions/2019-11-01-preview/sparkPools/testsparkpool/sessions/10/statements + uri: https://yanjuntestws001.dev.azuresynapse.net/livyApi/versions/2019-11-01-preview/sparkPools/sparkpooltest/sessions/4/statements response: body: - string: '{"id":0,"code":"import time\ntime.sleep(10)\nprint(''hello from cli'')","state":"waiting","output":null}' + string: '{"id":0,"code":"import time\ntime.sleep(10)\nprint(''hello from cli'')","state":"waiting","sourceId":"default","output":null}' headers: + content-length: + - '123' content-type: - application/json; charset=utf-8 date: - - Thu, 23 Jul 2020 04:31:40 GMT + - Mon, 24 Feb 2025 10:47:50 GMT server: - Kestrel Microsoft-HTTPAPI/2.0 strict-transport-security: - max-age=31536000; includeSubDomains - transfer-encoding: - - chunked x-ms-activity-id: - - 8fc98c55-80e0-4834-81a8-f8004bb6e543 - x-ms-response-time-ms: - - '45' + - 5d70c1c0-da02-4e9c-856e-80455f8c9874 status: code: 200 message: OK @@ -350,27 +239,25 @@ interactions: Connection: - keep-alive User-Agent: - - azsdk-python-synapse/0.2.0 Python/3.7.8 (Windows-10-10.0.18362-SP0) + - azsdk-python-synapse-spark/0.7.0 Python/3.10.11 (Windows-10-10.0.26100-SP0) method: GET - uri: https://testsynapseworkspace.dev.azuresynapse.net/livyApi/versions/2019-11-01-preview/sparkPools/testsparkpool/sessions/10/statements/0 + uri: https://yanjuntestws001.dev.azuresynapse.net/livyApi/versions/2019-11-01-preview/sparkPools/sparkpooltest/sessions/4/statements/0 response: body: - string: '{"id":0,"code":"import time\ntime.sleep(10)\nprint(''hello from cli'')","state":"running","output":null}' + string: '{"id":0,"code":"import time\ntime.sleep(10)\nprint(''hello from cli'')","state":"running","sourceId":"default","output":null}' headers: + content-length: + - '123' content-type: - application/json; charset=utf-8 date: - - Thu, 23 Jul 2020 04:31:51 GMT + - Mon, 24 Feb 2025 10:48:01 GMT server: - Kestrel Microsoft-HTTPAPI/2.0 strict-transport-security: - max-age=31536000; includeSubDomains - transfer-encoding: - - chunked x-ms-activity-id: - - b4f494bf-540e-437c-9d63-31cc50286985 - x-ms-response-time-ms: - - '22' + - 4308bcfb-2ef6-4a24-984c-3a8614ac6eb6 status: code: 200 message: OK @@ -384,28 +271,26 @@ interactions: Connection: - keep-alive User-Agent: - - azsdk-python-synapse/0.2.0 Python/3.7.8 (Windows-10-10.0.18362-SP0) + - azsdk-python-synapse-spark/0.7.0 Python/3.10.11 (Windows-10-10.0.26100-SP0) method: GET - uri: https://testsynapseworkspace.dev.azuresynapse.net/livyApi/versions/2019-11-01-preview/sparkPools/testsparkpool/sessions/10/statements + uri: https://yanjuntestws001.dev.azuresynapse.net/livyApi/versions/2019-11-01-preview/sparkPools/sparkpooltest/sessions/4/statements response: body: string: '{"total_statements":1,"statements":[{"id":0,"code":"import time\ntime.sleep(10)\nprint(''hello - from cli'')","state":"running","output":null}]}' + from cli'')","state":"running","sourceId":"default","output":null}]}' headers: + content-length: + - '161' content-type: - application/json; charset=utf-8 date: - - Thu, 23 Jul 2020 04:31:53 GMT + - Mon, 24 Feb 2025 10:48:02 GMT server: - Kestrel Microsoft-HTTPAPI/2.0 strict-transport-security: - max-age=31536000; includeSubDomains - transfer-encoding: - - chunked x-ms-activity-id: - - d70f705b-1ddd-410a-9c53-55a80735c811 - x-ms-response-time-ms: - - '77' + - 1f94d7cd-8399-49ba-bfb3-3a9df5305bc2 status: code: 200 message: OK @@ -421,27 +306,25 @@ interactions: Content-Length: - '0' User-Agent: - - azsdk-python-synapse/0.2.0 Python/3.7.8 (Windows-10-10.0.18362-SP0) + - azsdk-python-synapse-spark/0.7.0 Python/3.10.11 (Windows-10-10.0.26100-SP0) method: POST - uri: https://testsynapseworkspace.dev.azuresynapse.net/livyApi/versions/2019-11-01-preview/sparkPools/testsparkpool/sessions/10/statements/0/cancel + uri: https://yanjuntestws001.dev.azuresynapse.net/livyApi/versions/2019-11-01-preview/sparkPools/sparkpooltest/sessions/4/statements/0/cancel response: body: string: '{"msg":"canceled"}' headers: + content-length: + - '18' content-type: - application/json; charset=utf-8 date: - - Thu, 23 Jul 2020 04:31:54 GMT + - Mon, 24 Feb 2025 10:48:04 GMT server: - Kestrel Microsoft-HTTPAPI/2.0 strict-transport-security: - max-age=31536000; includeSubDomains - transfer-encoding: - - chunked x-ms-activity-id: - - 823f31b4-be97-486c-9dec-15d6da287c6b - x-ms-response-time-ms: - - '82' + - 4f5f22d2-64d7-464e-b099-54724faf590d status: code: 200 message: OK @@ -457,9 +340,9 @@ interactions: Content-Length: - '0' User-Agent: - - azsdk-python-synapse/0.2.0 Python/3.7.8 (Windows-10-10.0.18362-SP0) + - azsdk-python-synapse-spark/0.7.0 Python/3.10.11 (Windows-10-10.0.26100-SP0) method: DELETE - uri: https://testsynapseworkspace.dev.azuresynapse.net/livyApi/versions/2019-11-01-preview/sparkPools/testsparkpool/sessions/10 + uri: https://yanjuntestws001.dev.azuresynapse.net/livyApi/versions/2019-11-01-preview/sparkPools/sparkpooltest/sessions/4 response: body: string: '' @@ -467,15 +350,15 @@ interactions: content-length: - '0' date: - - Thu, 23 Jul 2020 04:31:55 GMT + - Mon, 24 Feb 2025 10:48:04 GMT server: - Kestrel Microsoft-HTTPAPI/2.0 strict-transport-security: - max-age=31536000; includeSubDomains x-ms-activity-id: - - 323d06eb-d34c-48eb-afad-b0fea48a8c1a + - 4f13cd04-4011-4982-b9ae-fff67615a0c1 x-ms-response-time-ms: - - '31' + - '37' status: code: 200 message: OK @@ -489,61 +372,50 @@ interactions: Connection: - keep-alive User-Agent: - - azsdk-python-synapse/0.2.0 Python/3.7.8 (Windows-10-10.0.18362-SP0) + - azsdk-python-synapse-spark/0.7.0 Python/3.10.11 (Windows-10-10.0.26100-SP0) method: GET - uri: https://testsynapseworkspace.dev.azuresynapse.net/livyApi/versions/2019-11-01-preview/sparkPools/testsparkpool/sessions/10?detailed=true + uri: https://yanjuntestws001.dev.azuresynapse.net/livyApi/versions/2019-11-01-preview/sparkPools/sparkpooltest/sessions/4?detailed=true response: body: - string: '{"livyInfo":{"notStartedAt":null,"startingAt":"2020-07-23T04:25:38.8804089+00:00","idleAt":"2020-07-23T04:26:04.0740324+00:00","deadAt":null,"shuttingDownAt":null,"killedAt":null,"recoveringAt":null,"busyAt":"2020-07-23T04:31:45.2153272+00:00","errorAt":null,"currentState":"killed","jobCreationRequest":{"name":"clisession000002","conf":{},"driverMemory":"28g","driverCores":4,"executorMemory":"28g","executorCores":4,"numExecutors":2}},"name":"clisession000002","workspaceName":"testsynapseworkspace","computeName":"testsparkpool","sparkPoolName":"testsparkpool","submitterName":"aaa@foo.com","submitterId":"6ae2ff4b-4939-4952-a097-d234aad383a3","artifactId":"Livy","jobType":"SparkSession","result":"Cancelled","schedulerInfo":{"submittedAt":"2020-07-23T04:25:33.75932+00:00","scheduledAt":"2020-07-23T04:25:35.1475929+00:00","endedAt":"2020-07-23T04:31:57.2070023+00:00","cancellationRequestedAt":"2020-07-23T04:31:55.4972535+00:00","currentState":"Ended"},"pluginInfo":{"preparationStartedAt":"2020-07-23T04:25:35.1590862+00:00","resourceAcquisitionStartedAt":"2020-07-23T04:25:35.5201647+00:00","submissionStartedAt":"2020-07-23T04:25:38.6883539+00:00","monitoringStartedAt":"2020-07-23T04:25:38.8953971+00:00","cleanupStartedAt":"2020-07-23T04:31:55.4972559+00:00","currentState":"Ended"},"tags":{},"id":10,"appId":"application_1595477936491_0002","appInfo":{"driverLogUrl":"http://da5952899ae247bbae2901a023b91b390006d921831:8042/node/containerlogs/container_1595477936491_0002_01_000001/trusted-service-user","sparkUiUrl":"http://da5952899ae247bbae2901a023b91b39003ef203634:8088/proxy/application_1595477936491_0002/"},"state":"killed","log":["\t - final status: UNDEFINED","\t tracking URL: http://da5952899ae247bbae2901a023b91b39003ef203634:8088/proxy/application_1595477936491_0002/","\t - user: trusted-service-user","20/07/23 04:25:43 INFO ShutdownHookManager: Shutdown - hook called","20/07/23 04:25:43 INFO ShutdownHookManager: Deleting directory - /tmp/spark-a26fc725-026a-4e91-9b8a-cb1994e2e6c8","20/07/23 04:25:43 INFO ShutdownHookManager: - Deleting directory /tmp/spark-0814d835-2f5f-45e5-a3cd-2d4a4695659d","20/07/23 - 04:25:43 INFO MetricsSystemImpl: Stopping azure-file-system metrics system...","20/07/23 - 04:25:43 INFO MetricsSystemImpl: azure-file-system metrics system stopped.","20/07/23 - 04:25:43 INFO MetricsSystemImpl: azure-file-system metrics system shutdown - complete.","\nYARN Diagnostics: "]}' + string: '{"livyInfo":{"notStartedAt":null,"startingAt":"2025-02-24T10:42:09.7963442+00:00","idleAt":"2025-02-24T10:42:41.9735224+00:00","deadAt":null,"shuttingDownAt":null,"killedAt":null,"recoveringAt":null,"busyAt":null,"errorAt":null,"currentState":"killed","jobCreationRequest":{"name":"clisession000002","conf":{},"driverMemory":"28g","driverCores":4,"executorMemory":"28g","executorCores":4,"numExecutors":2}},"name":"clisession000002","workspaceName":"yanjuntestws001","computeName":"sparkpooltest","sparkPoolName":"sparkpooltest","submitterName":"v-weishuzhao@microsoft.com","submitterId":"6a33200f-2b27-4fda-8567-b0c70d532441","artifactId":"Livy","jobType":"SparkSession","result":"Cancelled","schedulerInfo":{"submittedAt":"2025-02-24T10:41:45.6711287+00:00","queuedAt":"2025-02-24T10:41:45.8856562+00:00","scheduledAt":"2025-02-24T10:42:01.4498111+00:00","endedAt":"2025-02-24T10:48:35.0523496+00:00","cancellationRequestedAt":"2025-02-24T10:48:05.9596335+00:00","currentState":"Ended"},"pluginInfo":{"preparationStartedAt":"2025-02-24T10:42:03.460577+00:00","resourceAcquisitionStartedAt":"2025-02-24T10:42:05.4915055+00:00","submissionStartedAt":"2025-02-24T10:42:09.6944824+00:00","monitoringStartedAt":"2025-02-24T10:42:11.8224021+00:00","cleanupStartedAt":"2025-02-24T10:48:05.9633638+00:00","currentState":"Ended"},"tags":{},"id":4,"appId":"application_1740390690825_0004","appInfo":{"sparkUiUrl":"https://web.azuresynapse.net/sparkui/72f988bf-86f1-41af-91ab-2d7cd011db47/workspaces/yanjuntestws001/sparkpools/sparkpooltest/sessions/4/applications/application_1740390690825_0004","impulseErrorCode":null,"isStreamingQueryExists":"false","impulseClassification":null,"impulseTsg":null,"isSessionTimedOut":"false","driverLogUrl":"https://web.azuresynapse.net/sparkui/72f988bf-86f1-41af-91ab-2d7cd011db47/api/v1/workspaces/yanjuntestws001/sparkpools/sparkpooltest/sessions/4/applications/application_1740390690825_0004/driverlog/stderr"},"state":"killed","log":["stdout: + ","Picked up _JAVA_OPTIONS: -Djava.io.tmpdir=/mnt/tmp","Picked up _JAVA_OPTIONS: + -Djava.io.tmpdir=/mnt/tmp","\nstderr: ","\nYARN Diagnostics: "],"registeredSources":["default"]}' headers: + content-length: + - '2153' content-type: - application/json; charset=utf-8 date: - - Thu, 23 Jul 2020 04:33:56 GMT + - Mon, 24 Feb 2025 10:50:05 GMT server: - Kestrel Microsoft-HTTPAPI/2.0 strict-transport-security: - max-age=31536000; includeSubDomains - transfer-encoding: - - chunked x-ms-activity-id: - - 01bb4a52-4b3c-479d-8ca4-3b43ca737db8 - x-ms-job-cluster: - - https://vnet-efcd4013-eastus-3018-1.eastus.cloudapp.azure.com:58977/ + - 2300bd95-b52c-48a7-88c4-6cec67850361 x-ms-job-clusterrequested-on: - - 7/23/2020 4:25:35 AM +00:00 + - 1/1/0001 12:00:00 AM +00:00 x-ms-job-ended-on: - - 7/23/2020 4:31:57 AM +00:00 + - 2/24/2025 10:48:35 AM +00:00 x-ms-job-internal-id: - - '10' + - '4' x-ms-job-livysubmission-on: - - 7/23/2020 4:25:38 AM +00:00 + - 2/24/2025 10:42:09 AM +00:00 x-ms-job-queued-on: - - 7/23/2020 4:25:38 AM +00:00 + - 2/24/2025 10:42:07 AM +00:00 x-ms-job-result: - Cancelled x-ms-job-scheduled-on: - - 7/23/2020 4:25:35 AM +00:00 + - 2/24/2025 10:42:01 AM +00:00 x-ms-job-scheduler-state: - Ended x-ms-job-submitted-by-name: - - aaa@foo.com + - v-weishuzhao@microsoft.com x-ms-job-submitted-on: - - 7/23/2020 4:25:33 AM +00:00 + - 2/24/2025 10:41:45 AM +00:00 x-ms-job-type: - SparkServiceSession - x-ms-response-time-ms: - - '23' status: code: 200 message: OK -version: 1 \ No newline at end of file +version: 1 diff --git a/src/azure-cli/azure/cli/command_modules/synapse/tests/latest/test_synapse_scenario.py b/src/azure-cli/azure/cli/command_modules/synapse/tests/latest/test_synapse_scenario.py index 69d06c90762..e69cdfc288c 100644 --- a/src/azure-cli/azure/cli/command_modules/synapse/tests/latest/test_synapse_scenario.py +++ b/src/azure-cli/azure/cli/command_modules/synapse/tests/latest/test_synapse_scenario.py @@ -2038,15 +2038,15 @@ def test_ip_firewall_rules(self, resource_group, storage_account): @ResourceGroupPreparer(name_prefix='synapse-cli', random_name_length=16) def test_spark_job(self, resource_group): self.kwargs.update({ - 'spark-pool': 'testsparkpool', - 'workspace': 'testsynapseworkspace', + 'spark-pool': 'sparkpooltest', + 'workspace': 'yanjuntestws001', 'job': 'WordCount_Java', - 'main-definition-file': 'abfss://testfilesystem@adlsgen2account.dfs.core.windows.net/samples/java/wordcount/wordcount.jar', + 'main-definition-file': 'abfss://filesystem01@ywtestaccount.dfs.core.windows.net/wordcount/wordcount_2.11-0.1.jar', 'main-class-name': 'WordCount', 'arguments': [ - 'abfss://testfilesystem@adlsgen2account.dfs.core.windows.net/samples/java/wordcount/shakespeare.txt', - 'abfss://testfilesystem@adlsgen2account.dfs.core.windows.net/samples/java/wordcount/result/'], - 'executors': 2, + 'abfss://filesystem01@ywtestaccount.dfs.core.windows.net/wordcount/input/shakespeare.txt', + 'abfss://filesystem01@ywtestaccount.dfs.core.windows.net/wordcount/output/'], + 'executors': 4, 'executor-size': 'Medium', 'configuration': '{\\"spark.dynamicAllocation.maxExecutors\\":\\"18\\"}' }) @@ -2091,8 +2091,8 @@ def test_spark_job(self, resource_group): @ResourceGroupPreparer(name_prefix='synapse-cli', random_name_length=16) def test_spark_session_and_statements(self, resource_group): self.kwargs.update({ - 'spark-pool': 'testsparkpool', - 'workspace': 'testsynapseworkspace', + 'spark-pool': 'sparkpooltest', + 'workspace': 'yanjuntestws001', 'job': self.create_random_name(prefix='clisession', length=14), 'executor-size': 'Small', 'executors': 2, @@ -2161,10 +2161,7 @@ def test_spark_session_and_statements(self, resource_group): # cancel a spark session statement self.cmd('az synapse spark statement cancel --livy-id {statement-id} --session-id {session-id} ' - '--workspace-name {workspace} --spark-pool-name {spark-pool} --yes', - checks=[ - self.check('msg', 'canceled') - ]) + '--workspace-name {workspace} --spark-pool-name {spark-pool} --yes') # delete/cancel a spark session self.cmd('az synapse spark session cancel --livy-id {session-id} --workspace-name {workspace} ' diff --git a/src/azure-cli/requirements.py3.Darwin.txt b/src/azure-cli/requirements.py3.Darwin.txt index 80efc4ea126..97fd571aa33 100644 --- a/src/azure-cli/requirements.py3.Darwin.txt +++ b/src/azure-cli/requirements.py3.Darwin.txt @@ -84,9 +84,9 @@ azure-multiapi-storage==1.3.0 azure-nspkg==3.0.2 azure-storage-common==1.4.2 azure-synapse-accesscontrol==0.5.0 -azure-synapse-artifacts==0.19.0 +azure-synapse-artifacts==0.20.0 azure-synapse-managedprivateendpoints==0.4.0 -azure-synapse-spark==0.2.0 +azure-synapse-spark==0.7.0 bcrypt==3.2.0 certifi==2024.7.4 cffi==1.16.0 diff --git a/src/azure-cli/requirements.py3.Linux.txt b/src/azure-cli/requirements.py3.Linux.txt index 76123dab31a..3bd49f20970 100644 --- a/src/azure-cli/requirements.py3.Linux.txt +++ b/src/azure-cli/requirements.py3.Linux.txt @@ -84,9 +84,9 @@ azure-multiapi-storage==1.3.0 azure-nspkg==3.0.2 azure-storage-common==1.4.2 azure-synapse-accesscontrol==0.5.0 -azure-synapse-artifacts==0.19.0 +azure-synapse-artifacts==0.20.0 azure-synapse-managedprivateendpoints==0.4.0 -azure-synapse-spark==0.2.0 +azure-synapse-spark==0.7.0 bcrypt==3.2.0 certifi==2024.7.4 cffi==1.16.0 diff --git a/src/azure-cli/requirements.py3.windows.txt b/src/azure-cli/requirements.py3.windows.txt index e06b38a75cc..9290d5726f8 100644 --- a/src/azure-cli/requirements.py3.windows.txt +++ b/src/azure-cli/requirements.py3.windows.txt @@ -84,9 +84,9 @@ azure-multiapi-storage==1.3.0 azure-nspkg==3.0.2 azure-storage-common==1.4.2 azure-synapse-accesscontrol==0.5.0 -azure-synapse-artifacts==0.19.0 +azure-synapse-artifacts==0.20.0 azure-synapse-managedprivateendpoints==0.4.0 -azure-synapse-spark==0.2.0 +azure-synapse-spark==0.7.0 bcrypt==3.2.0 certifi==2024.7.4 cffi==1.16.0 diff --git a/src/azure-cli/setup.py b/src/azure-cli/setup.py index 23f6dcf5fa4..11edacc1575 100644 --- a/src/azure-cli/setup.py +++ b/src/azure-cli/setup.py @@ -126,9 +126,9 @@ 'azure-multiapi-storage~=1.3.0', 'azure-storage-common~=1.4', 'azure-synapse-accesscontrol~=0.5.0', - 'azure-synapse-artifacts~=0.19.0', + 'azure-synapse-artifacts~=0.20.0', 'azure-synapse-managedprivateendpoints~=0.4.0', - 'azure-synapse-spark~=0.2.0', + 'azure-synapse-spark~=0.7.0', 'chardet~=5.2.0', 'colorama~=0.4.4', # On Linux, the distribution (Ubuntu, Debian, etc) and version are checked for `az feedback`