local api (crowdsecurity#482)

Co-authored-by: AlteredCoder
Co-authored-by: erenJag

.github/workflows/ci_functests-install.yml

@@ -0,0 +1,90 @@
+name: Hub-CI
+
+on:
+  push:
+    branches:
+      - wip_lapi
+      - master
+  pull_request:
+    branches:
+      - wip_lapi
+      - master
+
+jobs:
+  build:
+    name: Install generated release and perform basic tests
+    runs-on: ubuntu-latest
+    steps:
+    - name: Set up Go 1.13
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.13
+      id: go
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v2
+    - name: Build release
+      run: BUILD_VERSION=xxx make release
+    - name: Install release
+      run: |
+        cd crowdsec-xxx
+        sudo bash -x ./wizard.sh --bininstall
+        sudo cscli machines add -a
+    - name: Post-installation check
+      run: |
+        sudo cscli hub update
+    - name: Install collection
+      run: |
+        sudo cscli hub list -a
+        sudo cscli parsers install crowdsecurity/syslog-logs crowdsecurity/sshd-logs crowdsecurity/dateparse-enrich
+        sudo cscli scenarios install crowdsecurity/ssh-bf
+    - name: Crowdsec start service
+      run: |
+        sudo systemctl start crowdsec
+    - name: Generate fake ssh bf logs
+      run: |
+        for i in `seq 1 10` ; do 
+          echo `date '+%b %d %H:%M:%S '`'sd-126005 sshd[12422]: Invalid user netflix from 1.1.1.172 port 35424' >> ssh-bf.log
+        done;
+    - name: Process ssh-bf logs in time-machine
+      run: |
+        sudo crowdsec -file ./ssh-bf.log -type syslog -no-api
+    - name: Cscli ban list check
+      #check that we got the expected ban and that the filters are working properly
+      run: |
+        sudo cscli decisions list
+        sudo cscli decisions list -o=json | jq -e '.[].decisions[0].value == "1.1.1.172"'
+        sudo cscli decisions list -r 1.1.1.0/24 -o=json | jq -e '.[].decisions[0].value == "1.1.1.172"'
+        sudo cscli decisions list -r 1.1.2.0/24 -o=json | jq -e '. == null'
+        sudo cscli decisions list -i 1.1.1.172 -o=json | jq -e '.[].decisions[0].value == "1.1.1.172"'
+        sudo cscli decisions list -i 1.1.1.173 -o=json | jq -e '. == null'
+    - name: Cscli ban del check
+      #check that the delete is working and that filters are working properly
+      run: |
+        sudo cscli decisions delete -i 1.1.1.173
+        sudo cscli decisions list -o=json | jq -e '.[].decisions[0].value == "1.1.1.172"'
+        sudo cscli decisions delete -i 1.1.1.172
+        sudo cscli decisions list -o=json | jq -e '. == null'
+    - name: Metrics check
+      run: |
+        sudo cscli metrics
+    - name: Service stop & config change
+      #shutdown the service, edit that acquisition.yaml
+      run: |
+        sudo systemctl stop crowdsec
+        echo "" | sudo tee -a /etc/crowdsec/acquis.yaml > /dev/null
+        echo "filename: /tmp/test.log" | sudo tee -a /etc/crowdsec/acquis.yaml > /dev/null
+        echo "labels:" | sudo tee -a /etc/crowdsec/acquis.yaml > /dev/null
+        echo "  type: syslog" | sudo tee -a /etc/crowdsec/acquis.yaml > /dev/null
+        touch /tmp/test.log
+    - name: Service start & check
+      run: |
+        sudo systemctl start crowdsec || sudo journalctl -xe
+    - name: Trigger events via normal acquisition
+      run: |
+        cat ssh-bf.log >> /tmp/test.log
+        sleep 1
+    - name: Check results
+      run: |
+        sudo cscli decisions list -o=json | jq -e '.[].decisions[0].value == "1.1.1.172"'
+
+

.github/workflows/ci_go-test.yml

@@ -27,9 +27,9 @@ jobs:
       uses: jandelgado/[email protected]
       with:
         infile: coverage.out
-        outfile: coverage.lcov
+        outfile: coverage.txt
     - name: Coveralls
       uses: coverallsapp/github-action@master
       with:
         github-token: ${{ secrets.GITHUB_TOKEN }}
-        path-to-lcov: coverage.lcov
+        path-to-lcov: coverage.txt

.github/workflows/ci_golangci-lint.yml

@@ -20,3 +20,5 @@ jobs:
           # Optional: golangci-lint command line arguments.
           args: --issues-exit-code=0 --timeout 5m
           only-new-issues: true
+
+

.github/workflows/hub_tests.yml → .github/workflows/ci_hub-tests.yml

@@ -1,4 +1,4 @@
-name: Hub tests
+name: Hub-CI
 
 on:
   push:
@@ -33,27 +33,26 @@ jobs:
         git clone https://github.com/crowdsecurity/hub-tests.git
         cd hub-tests
         make
-    - id: keydb
-      uses: pozetroninc/github-action-get-latest-release@master
+    - uses: oprypin/find-latest-tag@v1
       with:
-        owner: crowdsecurity
-        repo: crowdsec
-        excludes: prerelease, draft
+        repository: crowdsecurity/crowdsec  # The repository to scan.
+        releases-only: false  # We know that all relevant tags have a GitHub release for them.
+      id: crowdsec  # The step ID to refer to later.
     - name: Create crowdsec test env with all parsers from the release
       run: |
-        cd crowdsec-${{ steps.keydb.outputs.release }}
+        cd crowdsec-${{ steps.crowdsec.outputs.tag }}
         ./test_env.sh
         cd tests
         for i in `./cscli -c dev.yaml list parsers -a -o json | jq -r ".[].name" ` ; do 
           ./cscli -c dev.yaml install parser $i ; 
         done
     - name: Setup hub ci in crowdsec
-      working-directory: ./crowdsec-${{ steps.keydb.outputs.release }}/tests/
+      working-directory: ./crowdsec-${{ steps.crowdsec.outputs.tag }}/tests/
       run: |
         cp -R ../../hub-tests/tests .
         cp ../../hub-tests/main .
     - name: Run the HUB CI
-      working-directory: ./crowdsec-${{ steps.keydb.outputs.release }}/tests/
+      working-directory: ./crowdsec-${{ steps.crowdsec.outputs.tag }}/tests/
       run: |
         for i in `find ./tests -mindepth 1 -maxdepth 1 -type d` ; do 
           echo "::group::Test-${i}" ; 

Dockerfile

@@ -0,0 +1,22 @@
+ARG GOVERSION=1.14
+
+FROM golang:${GOVERSION}-alpine AS build
+
+WORKDIR /go/src/crowdsec
+COPY . .
+
+RUN apk update && apk add git jq gcc libc-dev make bash gettext
+RUN BUILD_VERSION="$(git describe --tags `git rev-list --tags --max-count=1`)" make release
+RUN /bin/bash wizard.sh --docker-mode
+RUN cscli hub update && cscli collections install crowdsecurity/linux
+
+FROM alpine:latest
+COPY --from=build /etc/crowdsec /etc/crowdsec
+COPY --from=build /var/lib/crowdsec /var/lib/crowdsec
+COPY --from=build /usr/local/bin/crowdsec /usr/local/bin/crowdsec
+COPY --from=build /usr/local/bin/cscli /usr/local/bin/cscli
+COPY --from=build /go/src/crowdsec/docker/docker_start.sh /
+COPY --from=build /go/src/crowdsec/docker/config.yaml /etc/crowdsec/config.yaml
+
+
+ENTRYPOINT /bin/sh docker_start.sh

Makefile

@@ -3,7 +3,6 @@ CFG_PREFIX = $(PREFIX)"/etc/crowdsec/"
 BIN_PREFIX = $(PREFIX)"/usr/local/bin/"
 DATA_PREFIX = $(PREFIX)"/var/run/crowdsec/"
 
-PLUGIN_FOLDER="./plugins"
 PID_DIR = $(PREFIX)"/var/run/"
 CROWDSEC_FOLDER = "./cmd/crowdsec"
 CSCLI_FOLDER = "./cmd/crowdsec-cli/"
@@ -14,14 +13,13 @@ BUILD_CMD="build"
 GOARCH=amd64
 GOOS=linux
 
-
-#Current versioning information from env
+#Golang version info
 GO_MAJOR_VERSION = $(shell go version | cut -c 14- | cut -d' ' -f1 | cut -d'.' -f1)
 GO_MINOR_VERSION = $(shell go version | cut -c 14- | cut -d' ' -f1 | cut -d'.' -f2)
 MINIMUM_SUPPORTED_GO_MAJOR_VERSION = 1
 MINIMUM_SUPPORTED_GO_MINOR_VERSION = 13
 GO_VERSION_VALIDATION_ERR_MSG = Golang version ($(BUILD_GOVERSION)) is not supported, please use least $(MINIMUM_SUPPORTED_GO_MAJOR_VERSION).$(MINIMUM_SUPPORTED_GO_MINOR_VERSION)
-
+#Current versioning information from env
 BUILD_VERSION?="$(shell git describe --tags `git rev-list --tags --max-count=1`)"
 BUILD_GOVERSION="$(shell go version | cut -d " " -f3 | sed -r 's/[go]+//g')"
 BUILD_CODENAME=$(shell cat RELEASE.json | jq -r .CodeName)
@@ -36,20 +34,20 @@ RELDIR = crowdsec-$(BUILD_VERSION)
 
 all: clean test build
 
-build: clean goversion crowdsec cscli
+build: goversion crowdsec cscli
 
 static: goversion crowdsec_static cscli_static
 
 goversion:
 	@if [ $(GO_MAJOR_VERSION) -gt $(MINIMUM_SUPPORTED_GO_MAJOR_VERSION) ]; then \
-		exit 0 ;\
-	elif [ $(GO_MAJOR_VERSION) -lt $(MINIMUM_SUPPORTED_GO_MAJOR_VERSION) ]; then \
-		echo '$(GO_VERSION_VALIDATION_ERR_MSG)';\
-		exit 1; \
-	elif [ $(GO_MINOR_VERSION) -lt $(MINIMUM_SUPPORTED_GO_MINOR_VERSION) ] ; then \
-		echo '$(GO_VERSION_VALIDATION_ERR_MSG)';\
-		exit 1; \
-	fi
+        exit 0 ;\
+    elif [ $(GO_MAJOR_VERSION) -lt $(MINIMUM_SUPPORTED_GO_MAJOR_VERSION) ]; then \
+        echo '$(GO_VERSION_VALIDATION_ERR_MSG)';\
+        exit 1; \
+    elif [ $(GO_MINOR_VERSION) -lt $(MINIMUM_SUPPORTED_GO_MINOR_VERSION) ] ; then \
+        echo '$(GO_VERSION_VALIDATION_ERR_MSG)';\
+        exit 1; \
+    fi
 
 hubci:
 	@rm -rf crowdsec-xxx hub-tests
@@ -69,26 +67,50 @@ clean:
 	@rm -f $(CSCLI_BIN)
 	@rm -f *.log
 
-cscli: goversion
+cscli:
+ifeq ($(lastword $(RESPECT_VERSION)), $(CURRENT_GOVERSION))
 	@make -C $(CSCLI_FOLDER) build --no-print-directory
+else
+	@echo "Required golang version is $(REQUIRE_GOVERSION). The current one is $(CURRENT_GOVERSION). Exiting.."
+	@exit 1;
+endif
 
 
-
-crowdsec: goversion
+crowdsec:
+ifeq ($(lastword $(RESPECT_VERSION)), $(CURRENT_GOVERSION))
 	@make -C $(CROWDSEC_FOLDER) build --no-print-directory
-	@bash ./scripts/build_plugins.sh
+else
+	@echo "Required golang version is $(REQUIRE_GOVERSION). The current one is $(CURRENT_GOVERSION). Exiting.."
+	@exit 1;
+endif
 
 
-cscli_static: goversion
+cscli_static:
+ifeq ($(lastword $(RESPECT_VERSION)), $(CURRENT_GOVERSION))
 	@make -C $(CSCLI_FOLDER) static --no-print-directory
+else
+	@echo "Required golang version is $(REQUIRE_GOVERSION). The current one is $(CURRENT_GOVERSION). Exiting.."
+	@exit 1;
+endif
 
 
-crowdsec_static: goversion
+crowdsec_static:
+ifeq ($(lastword $(RESPECT_VERSION)), $(CURRENT_GOVERSION))
 	@make -C $(CROWDSEC_FOLDER) static --no-print-directory
+else
+	@echo "Required golang version is $(REQUIRE_GOVERSION). The current one is $(CURRENT_GOVERSION). Exiting.."
+	@exit 1;
+endif
+
 
 #.PHONY: test
 test:
+ifeq ($(lastword $(RESPECT_VERSION)), $(CURRENT_GOVERSION))
 	@make -C $(CROWDSEC_FOLDER) test --no-print-directory
+else
+	@echo "Required golang version is $(REQUIRE_GOVERSION). The current one is $(CURRENT_GOVERSION). Exiting.."
+	@exit 1;
+endif
 
 .PHONY: uninstall
 uninstall:
@@ -110,7 +132,4 @@ release: check_release build
 	@cp -R ./config/ $(RELDIR)
 	@cp wizard.sh $(RELDIR)
 	@cp scripts/test_env.sh $(RELDIR)
-	@bash ./scripts/build_plugins.sh
-	@mkdir -p "$(RELDIR)/plugins/backend"
-	@find ./plugins -type f -name "*.so" -exec install -Dm 644 {} "$(RELDIR)/{}" \; || exiting 
 	@tar cvzf crowdsec-release.tgz $(RELDIR)