From 2043d80f82dc60d9abde77ded4c92707a660874e Mon Sep 17 00:00:00 2001 From: Saylor Berman Date: Thu, 21 Nov 2024 09:07:33 -0700 Subject: [PATCH] Adding logs to find issue --- .github/workflows/ci.yml | 236 +++++++++--------- internal/mode/static/handler.go | 46 ++++ internal/mode/static/manager.go | 1 + .../mode/static/status/prepare_requests.go | 6 +- .../static/status/prepare_requests_test.go | 1 + 5 files changed, 170 insertions(+), 120 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e14b54ed68..4efc4d8147 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,62 +74,62 @@ jobs: - name: Check if all the generated files are up to date run: make generate-all && git diff --exit-code - unit-tests: - name: Unit Tests - runs-on: ubuntu-24.04 - needs: vars - steps: - - name: Checkout Repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Setup Golang Environment - uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0 - with: - go-version: stable - cache-dependency-path: | - go.sum - .github/.cache/buster-for-unit-tests - - - name: Run Tests - run: make unit-test - - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0 - with: - token: ${{ secrets.CODECOV_TOKEN }} - - - name: Upload Coverage Report - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 - with: - name: cover-${{ github.run_id }}.html - path: ${{ github.workspace }}/cover.html - if: always() - - njs-unit-tests: - name: NJS Unit Tests - runs-on: ubuntu-24.04 - needs: vars - steps: - - name: Checkout Repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Setup Node.js Environment - uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0 - with: - node-version-file: .nvmrc - - - name: Run tests - run: npm --prefix ${{ github.workspace }}/internal/mode/static/nginx/modules install-ci-test - - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0 - with: - token: ${{ secrets.CODECOV_TOKEN }} + # unit-tests: + # name: Unit Tests + # runs-on: ubuntu-24.04 + # needs: vars + # steps: + # - name: Checkout Repository + # uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + # - name: Setup Golang Environment + # uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0 + # with: + # go-version: stable + # cache-dependency-path: | + # go.sum + # .github/.cache/buster-for-unit-tests + + # - name: Run Tests + # run: make unit-test + + # - name: Upload coverage reports to Codecov + # uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0 + # with: + # token: ${{ secrets.CODECOV_TOKEN }} + + # - name: Upload Coverage Report + # uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + # with: + # name: cover-${{ github.run_id }}.html + # path: ${{ github.workspace }}/cover.html + # if: always() + + # njs-unit-tests: + # name: NJS Unit Tests + # runs-on: ubuntu-24.04 + # needs: vars + # steps: + # - name: Checkout Repository + # uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + # - name: Setup Node.js Environment + # uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0 + # with: + # node-version-file: .nvmrc + + # - name: Run tests + # run: npm --prefix ${{ github.workspace }}/internal/mode/static/nginx/modules install-ci-test + + # - name: Upload coverage reports to Codecov + # uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0 + # with: + # token: ${{ secrets.CODECOV_TOKEN }} binary: name: Build Binary runs-on: ubuntu-24.04 - needs: [vars, unit-tests, njs-unit-tests] + needs: [vars] permissions: contents: write # for goreleaser/goreleaser-action and lucacome/draft-release to create/update releases id-token: write # for goreleaser/goreleaser-action to sign artifacts @@ -222,25 +222,25 @@ jobs: id-token: write # for docker/login to login to NGINX registry secrets: inherit - functional-tests: - name: Functional tests - needs: [vars, build-oss, build-plus] - strategy: - fail-fast: false - matrix: - image: [nginx, plus] - k8s-version: - [ - "${{ needs.vars.outputs.min_k8s_version }}", - "${{ needs.vars.outputs.k8s_latest }}", - ] - uses: ./.github/workflows/functional.yml - with: - image: ${{ matrix.image }} - k8s-version: ${{ matrix.k8s-version }} - secrets: inherit - permissions: - contents: read + # functional-tests: + # name: Functional tests + # needs: [vars, build-oss, build-plus] + # strategy: + # fail-fast: false + # matrix: + # image: [nginx, plus] + # k8s-version: + # [ + # "${{ needs.vars.outputs.min_k8s_version }}", + # "${{ needs.vars.outputs.k8s_latest }}", + # ] + # uses: ./.github/workflows/functional.yml + # with: + # image: ${{ matrix.image }} + # k8s-version: ${{ matrix.k8s-version }} + # secrets: inherit + # permissions: + # contents: read conformance-tests: name: Conformance tests @@ -264,50 +264,50 @@ jobs: permissions: contents: write - helm-tests: - name: Helm Tests - needs: [vars, build-oss, build-plus] - strategy: - fail-fast: false - matrix: - image: [nginx, plus] - k8s-version: - [ - "${{ needs.vars.outputs.min_k8s_version }}", - "${{ needs.vars.outputs.k8s_latest }}", - ] - uses: ./.github/workflows/helm.yml - with: - image: ${{ matrix.image }} - k8s-version: ${{ matrix.k8s-version }} - secrets: inherit - if: ${{ needs.vars.outputs.helm_changes == 'true' || github.event_name == 'schedule' }} - - publish-helm: - name: Package and Publish Helm Chart - runs-on: ubuntu-24.04 - needs: [vars, helm-tests] - if: ${{ github.event_name == 'push' && ! startsWith(github.ref, 'refs/heads/release-') }} - permissions: - contents: read - packages: write # for helm to push to GHCR - steps: - - name: Checkout Repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Login to GitHub Container Registry - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Package - id: package - run: | - output=$(helm package ${{ github.ref_type != 'tag' && '--app-version edge --version 0.0.0-edge' || '' }} charts/nginx-gateway-fabric) - echo "path=$(basename -- $(echo $output | cut -d: -f2))" >> $GITHUB_OUTPUT - - - name: Push to GitHub Container Registry - run: | - helm push ${{ steps.package.outputs.path }} oci://ghcr.io/nginxinc/charts + # helm-tests: + # name: Helm Tests + # needs: [vars, build-oss, build-plus] + # strategy: + # fail-fast: false + # matrix: + # image: [nginx, plus] + # k8s-version: + # [ + # "${{ needs.vars.outputs.min_k8s_version }}", + # "${{ needs.vars.outputs.k8s_latest }}", + # ] + # uses: ./.github/workflows/helm.yml + # with: + # image: ${{ matrix.image }} + # k8s-version: ${{ matrix.k8s-version }} + # secrets: inherit + # if: ${{ needs.vars.outputs.helm_changes == 'true' || github.event_name == 'schedule' }} + + # publish-helm: + # name: Package and Publish Helm Chart + # runs-on: ubuntu-24.04 + # needs: [vars, helm-tests] + # if: ${{ github.event_name == 'push' && ! startsWith(github.ref, 'refs/heads/release-') }} + # permissions: + # contents: read + # packages: write # for helm to push to GHCR + # steps: + # - name: Checkout Repository + # uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + # - name: Login to GitHub Container Registry + # uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + # with: + # registry: ghcr.io + # username: ${{ github.repository_owner }} + # password: ${{ secrets.GITHUB_TOKEN }} + + # - name: Package + # id: package + # run: | + # output=$(helm package ${{ github.ref_type != 'tag' && '--app-version edge --version 0.0.0-edge' || '' }} charts/nginx-gateway-fabric) + # echo "path=$(basename -- $(echo $output | cut -d: -f2))" >> $GITHUB_OUTPUT + + # - name: Push to GitHub Container Registry + # run: | + # helm push ${{ steps.package.outputs.path }} oci://ghcr.io/nginxinc/charts diff --git a/internal/mode/static/handler.go b/internal/mode/static/handler.go index 637dc378c7..ce6fb04d6c 100644 --- a/internal/mode/static/handler.go +++ b/internal/mode/static/handler.go @@ -1,6 +1,7 @@ package static import ( + "bytes" "context" "fmt" "sync" @@ -11,6 +12,8 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" @@ -38,6 +41,7 @@ type handlerMetricsCollector interface { // eventHandlerConfig holds configuration parameters for eventHandlerImpl. type eventHandlerConfig struct { + k8sConfig *rest.Config // nginxFileMgr is the file Manager for nginx. nginxFileMgr file.Manager // metricsCollector collects metrics for this controller. @@ -263,6 +267,11 @@ func (h *eventHandlerImpl) updateStatuses(ctx context.Context, logger logr.Logge h.cfg.statusUpdater.UpdateGroup(ctx, groupAllExceptGateways, reqs...) + logs, err := h.GetPodLogs() + if err != nil { + logger.Error(err, "getting logs") + } + // We put Gateway status updates separately from the rest of the statuses because we want to be able // to update them separately from the rest of the graph whenever the public IP of NGF changes. gwReqs := status.PrepareGatewayRequests( @@ -271,10 +280,45 @@ func (h *eventHandlerImpl) updateStatuses(ctx context.Context, logger logr.Logge transitionTime, gwAddresses, h.latestReloadResult, + logs, ) h.cfg.statusUpdater.UpdateGroup(ctx, groupGateways, gwReqs...) } +func (h *eventHandlerImpl) GetPodLogs() (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + namespace := h.cfg.gatewayPodConfig.Namespace + name := h.cfg.gatewayPodConfig.Name + opts := &v1.PodLogOptions{ + Container: "nginx", + TailLines: helpers.GetPointer(int64(20)), + } + + clientGoClient, err := kubernetes.NewForConfig(h.cfg.k8sConfig) + if err != nil { + return err.Error(), err + } + + req := clientGoClient.CoreV1().Pods(namespace).GetLogs(name, opts) + + logs, err := req.Stream(ctx) + if err != nil { + return fmt.Sprintf("error getting logs from Pod: %v", err), + fmt.Errorf("error getting logs from Pod: %w", err) + } + defer logs.Close() + + buf := new(bytes.Buffer) + if _, err := buf.ReadFrom(logs); err != nil { + return fmt.Sprintf("error reading logs from Pod: %v", err), + fmt.Errorf("error reading logs from Pod: %w", err) + } + + return buf.String(), nil +} + func (h *eventHandlerImpl) parseAndCaptureEvent(ctx context.Context, logger logr.Logger, event interface{}) { switch e := event.(type) { case *events.UpsertEvent: @@ -610,6 +654,7 @@ func (h *eventHandlerImpl) nginxGatewayServiceUpsert(ctx context.Context, logger transitionTime, gwAddresses, h.latestReloadResult, + "", ) h.cfg.statusUpdater.UpdateGroup(ctx, groupGateways, gatewayStatuses...) } @@ -636,6 +681,7 @@ func (h *eventHandlerImpl) nginxGatewayServiceDelete( transitionTime, gwAddresses, h.latestReloadResult, + "", ) h.cfg.statusUpdater.UpdateGroup(ctx, groupGateways, gatewayStatuses...) } diff --git a/internal/mode/static/manager.go b/internal/mode/static/manager.go index 4c80fc4260..c5a4b6ce46 100644 --- a/internal/mode/static/manager.go +++ b/internal/mode/static/manager.go @@ -215,6 +215,7 @@ func StartManager(cfg config.Config) error { groupStatusUpdater := status.NewLeaderAwareGroupUpdater(statusUpdater) eventHandler := newEventHandlerImpl(eventHandlerConfig{ + k8sConfig: mgr.GetConfig(), k8sClient: mgr.GetClient(), k8sReader: mgr.GetAPIReader(), processor: processor, diff --git a/internal/mode/static/status/prepare_requests.go b/internal/mode/static/status/prepare_requests.go index e0add956a8..f0b1a3c4b9 100644 --- a/internal/mode/static/status/prepare_requests.go +++ b/internal/mode/static/status/prepare_requests.go @@ -215,11 +215,12 @@ func PrepareGatewayRequests( transitionTime metav1.Time, gwAddresses []v1.GatewayStatusAddress, nginxReloadRes NginxReloadResult, + logs string, ) []frameworkStatus.UpdateRequest { reqs := make([]frameworkStatus.UpdateRequest, 0, 1+len(ignoredGateways)) if gateway != nil { - reqs = append(reqs, prepareGatewayRequest(gateway, transitionTime, gwAddresses, nginxReloadRes)) + reqs = append(reqs, prepareGatewayRequest(gateway, transitionTime, gwAddresses, nginxReloadRes, logs)) } for nsname, gw := range ignoredGateways { @@ -241,6 +242,7 @@ func prepareGatewayRequest( transitionTime metav1.Time, gwAddresses []v1.GatewayStatusAddress, nginxReloadRes NginxReloadResult, + logs string, ) frameworkStatus.UpdateRequest { if !gateway.Valid { conds := conditions.ConvertConditions( @@ -302,7 +304,7 @@ func prepareGatewayRequest( if nginxReloadRes.Error != nil { gwConds = append( gwConds, - staticConds.NewGatewayNotProgrammedInvalid(staticConds.GatewayMessageFailedNginxReload), + staticConds.NewGatewayNotProgrammedInvalid(staticConds.GatewayMessageFailedNginxReload+logs), ) } diff --git a/internal/mode/static/status/prepare_requests_test.go b/internal/mode/static/status/prepare_requests_test.go index d52b43e7a8..558c309609 100644 --- a/internal/mode/static/status/prepare_requests_test.go +++ b/internal/mode/static/status/prepare_requests_test.go @@ -1166,6 +1166,7 @@ func TestBuildGatewayStatuses(t *testing.T) { transitionTime, addr, test.nginxReloadRes, + "", ) g.Expect(reqs).To(HaveLen(expectedTotalReqs))