From 2ca4b1194840147ed80cb11262ef9637cb4c8180 Mon Sep 17 00:00:00 2001 From: Pavel Dotsulenko Date: Mon, 28 Oct 2019 23:17:59 +0200 Subject: [PATCH 1/4] Report job error when worker fails to unmarshal job config --- amqp_job_queue.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/amqp_job_queue.go b/amqp_job_queue.go index b9ace1c7c..14491f065 100644 --- a/amqp_job_queue.go +++ b/amqp_job_queue.go @@ -176,6 +176,7 @@ func (q *AMQPJobQueue) Jobs(ctx gocontext.Context) (outChan <-chan Job, err erro startAttributes: &backend.StartAttributes{}, stateUpdatePool: q.stateUpdatePool, withLogSharding: q.withLogSharding, + logWriterChan: logWriterChannel, } startAttrs := &jobPayloadStartAttrs{Config: &backend.StartAttributes{}} @@ -193,11 +194,16 @@ func (q *AMQPJobQueue) Jobs(ctx gocontext.Context) (outChan <-chan Job, err erro err = json.Unmarshal(delivery.Body, &startAttrs) if err != nil { - logger.WithField("err", err).Error("start attributes JSON parse error, attempting to ack+drop delivery") + logger.WithField("err", err). + WithField("json", string(delivery.Body)). + Error("start attributes JSON parse error, attempting to ack+drop delivery") + err := delivery.Ack(false) if err != nil { logger.WithField("err", err).WithField("delivery", delivery).Error("couldn't ack+drop delivery") } + + buildJob.Error(ctx, "An error occurred while parsing the job config.") continue } @@ -217,7 +223,6 @@ func (q *AMQPJobQueue) Jobs(ctx gocontext.Context) (outChan <-chan Job, err erro buildJob.startAttributes.Warmer = buildJob.payload.Warmer buildJob.startAttributes.SetDefaults(q.DefaultLanguage, q.DefaultDist, q.DefaultArch, q.DefaultGroup, q.DefaultOS, VMTypeDefault, VMConfigDefault) buildJob.conn = q.conn - buildJob.logWriterChan = logWriterChannel buildJob.delivery = delivery buildJob.stateCount = buildJob.payload.Meta.StateUpdateCount From 19021d258f9a32ebfd1d9d3b1abfc6828b6cbab5 Mon Sep 17 00:00:00 2001 From: Pavel Dotsulenko Date: Mon, 28 Oct 2019 23:32:02 +0200 Subject: [PATCH 2/4] Add error check --- amqp_job_queue.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/amqp_job_queue.go b/amqp_job_queue.go index 14491f065..59bae68ed 100644 --- a/amqp_job_queue.go +++ b/amqp_job_queue.go @@ -203,7 +203,11 @@ func (q *AMQPJobQueue) Jobs(ctx gocontext.Context) (outChan <-chan Job, err erro logger.WithField("err", err).WithField("delivery", delivery).Error("couldn't ack+drop delivery") } - buildJob.Error(ctx, "An error occurred while parsing the job config.") + err = buildJob.Error(ctx, "An error occurred while parsing the job config.") + if err != nil { + logger.WithField("err", err).Error("couldn't error the job") + } + continue } From eb9bd0b3268158208f5be56ef358207082a0cdfa Mon Sep 17 00:00:00 2001 From: Pavel Dotsulenko Date: Tue, 29 Oct 2019 13:00:03 +0200 Subject: [PATCH 3/4] Initialize delivery. Do not log raw JSON job config, because it may contain sensitive info --- amqp_job_queue.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/amqp_job_queue.go b/amqp_job_queue.go index 59bae68ed..24fc43c10 100644 --- a/amqp_job_queue.go +++ b/amqp_job_queue.go @@ -177,6 +177,7 @@ func (q *AMQPJobQueue) Jobs(ctx gocontext.Context) (outChan <-chan Job, err erro stateUpdatePool: q.stateUpdatePool, withLogSharding: q.withLogSharding, logWriterChan: logWriterChannel, + delivery: delivery, } startAttrs := &jobPayloadStartAttrs{Config: &backend.StartAttributes{}} @@ -194,9 +195,7 @@ func (q *AMQPJobQueue) Jobs(ctx gocontext.Context) (outChan <-chan Job, err erro err = json.Unmarshal(delivery.Body, &startAttrs) if err != nil { - logger.WithField("err", err). - WithField("json", string(delivery.Body)). - Error("start attributes JSON parse error, attempting to ack+drop delivery") + logger.WithField("err", err).Error("start attributes JSON parse error, attempting to ack+drop delivery") err := delivery.Ack(false) if err != nil { @@ -227,7 +226,6 @@ func (q *AMQPJobQueue) Jobs(ctx gocontext.Context) (outChan <-chan Job, err erro buildJob.startAttributes.Warmer = buildJob.payload.Warmer buildJob.startAttributes.SetDefaults(q.DefaultLanguage, q.DefaultDist, q.DefaultArch, q.DefaultGroup, q.DefaultOS, VMTypeDefault, VMConfigDefault) buildJob.conn = q.conn - buildJob.delivery = delivery buildJob.stateCount = buildJob.payload.Meta.StateUpdateCount jobSendBegin := time.Now() From 32c3a72a7331439619865f4e4ae8044d6482ad1e Mon Sep 17 00:00:00 2001 From: Pavel Dotsulenko Date: Tue, 29 Oct 2019 17:20:28 +0200 Subject: [PATCH 4/4] Improve error message --- amqp_job_queue.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/amqp_job_queue.go b/amqp_job_queue.go index 24fc43c10..39025277d 100644 --- a/amqp_job_queue.go +++ b/amqp_job_queue.go @@ -202,7 +202,7 @@ func (q *AMQPJobQueue) Jobs(ctx gocontext.Context) (outChan <-chan Job, err erro logger.WithField("err", err).WithField("delivery", delivery).Error("couldn't ack+drop delivery") } - err = buildJob.Error(ctx, "An error occurred while parsing the job config.") + err = buildJob.Error(ctx, "An error occured while parsing the job config. Please consider enabling the build config validation feature for the repository: https://docs.travis-ci.com/user/build-config-validation") if err != nil { logger.WithField("err", err).Error("couldn't error the job") }