Merge branch 'master' into dependabot/pip/requirements/pyyaml-5.4

.github/workflows/cleanup_acceptance.yaml

@@ -10,6 +10,8 @@ jobs:
     container: quay.io/hypernode/deploy:3-php8.1-node18
     steps:
       - uses: actions/checkout@v2
+      - name: Dump env
+        run: env
       - name: Cleanup acceptance environments
         run: hypernode-deploy cleanup acceptance -vvv
         env:

.github/workflows/deploy.yaml

@@ -14,15 +14,23 @@ jobs:
     runs-on: ubuntu-latest
     container: quay.io/hypernode/deploy:3-php8.1-node18
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      # Declares the repository safe and not under dubious ownership.
+      - name: Add repository to git safe directories
+        run: git config --global --add safe.directory $GITHUB_WORKSPACE
       - uses: actions/cache@v2
         with:
           path: /tmp/composer-cache
           key: ${{ runner.os }}-composer
       - uses: webfactory/[email protected]
         with:
           ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
-      - run: apt update && apt install -y python3 virtualenv
+      - name: Set env for production build
+        if: github.ref == 'refs/heads/master'
+        run: |
+          echo "DOCS_BASE_URL=https://docs.hypernode.com/" >> $GITHUB_ENV
       - run: hypernode-deploy build -vvv
       - name: archive production artifacts
         uses: actions/upload-artifact@v3
@@ -33,6 +41,9 @@ jobs:
   deploy_acceptance:
     needs: build
     runs-on: ubuntu-latest
+    environment:
+      name: acceptance
+      url: ${{ steps.get_brancher_hostname.outputs.BRANCHER_URL }}
     if: github.ref != 'refs/heads/master'
     container: quay.io/hypernode/deploy:3-php8.1-node18
     steps:
@@ -47,20 +58,24 @@ jobs:
           ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
       - run: mkdir -p $HOME/.ssh
       - name: deploy to acceptance
-        run: hypernode-deploy deploy acceptance -vvv
+        run: hypernode-deploy deploy acceptance -vvv --reuse-brancher
         env:
           HYPERNODE_API_TOKEN: ${{ secrets.HYPERNODE_API_TOKEN }}
       - name: Get brancher hostname
-        run: echo "BRANCHER_HOSTNAME=$(jq .hostnames[0] deployment-report.json -r)" >> $GITHUB_ENV
+        id: get_brancher_hostname
+        run: echo "BRANCHER_URL=https://$(jq .hostnames[0] deployment-report.json -r)" >> $GITHUB_OUTPUT
       - name: Comment hostname on PR
         uses: thollander/actions-comment-pull-request@v1
         with:
           message: |
-            Acceptance server is available at https://${{ env.BRANCHER_HOSTNAME }}
+            Acceptance server is available at ${{ steps.get_brancher_hostname.outputs.BRANCHER_URL }}
 
   deploy_production:
     needs: build
     runs-on: ubuntu-latest
+    environment:
+      name: production
+      url: https://docs.hypernode.io
     if: github.ref == 'refs/heads/master'
     container: quay.io/hypernode/deploy:3-php8.1-node18
     steps:

.github/workflows/detect_renamed_docs.yaml

@@ -0,0 +1,27 @@
+name: Detect renamed docs
+
+on:
+  pull_request:
+
+jobs:
+  detect_renamed_docs:
+    runs-on: ubuntu-latest
+    name: Detect renamed docs
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Get changed files
+        id: changed_files
+        run: |
+          result="$(python3 ci/bin/detect_rewrites.py ${{ github.event.pull_request.base.sha }} ${{github.event.pull_request.head.sha}})"
+          echo "$result"
+          echo "CHANGED_FILES<<EOF" >> $GITHUB_OUTPUT
+          echo "$result" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+        shell: bash
+      - name: Comment hostname on PR
+        if: ${{ steps.changed_files.outputs.CHANGED_FILES }}
+        uses: thollander/actions-comment-pull-request@v1
+        with:
+          message: "${{ steps.changed_files.outputs.CHANGED_FILES }}"

.pre-commit-config.yaml

@@ -2,21 +2,22 @@
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: 'v4.1.0'
+    rev: 'v4.4.0'
     hooks:
     -   id: trailing-whitespace
     -   id: end-of-file-fixer
 -   repo: https://github.com/psf/black
-    rev: '22.10.0'
+    rev: '22.12.0'
     hooks:
     -   id: black
 -   repo: https://github.com/myint/autoflake
-    rev: 'v1.4'
+    rev: 'v2.0.0'
     hooks:
     -   id: autoflake
         args: ["--in-place", "--remove-unused-variables", "--remove-all-unused-imports"]
 -   repo: https://github.com/pycqa/isort
-    rev: '5.10.1'
+    rev: '5.11.4'
     hooks:
     -   id: isort
         name: isort (python)
+        args: ["--profile", "black"]

LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Hypernode
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -66,3 +66,48 @@ Then deploy to your local Hypernode Docker:
 docker run --rm -it --env SSH_PRIVATE_KEY="$(cat ~/.ssh/yourdeploykey | base64)" -v ${PWD}:/build quay.io/hypernode/deploy:latest hypernode-deploy build -vvv  # First build the artifact
 docker run --rm -it --env SSH_PRIVATE_KEY="$(cat ~/.ssh/yourdeploykey | base64)" -v ${PWD}:/build quay.io/hypernode/deploy:latest hypernode-deploy deploy docker -vvv  # Then perform the deploy
 ```
+
+## Building the manpage deb
+
+The docs are also packaged as a debian package named `hndocsnext` so that on a Hypernode you can run `man hypernode` (or `hypernode-manual`) and page through a `manpage` version of the Hypernode docs. To build that debian package on a Debian machine you can run these commands:
+```
+# First create the cow environment
+export ARCH=amd64
+export DIST=buster
+apt-get install debhelper cowbuilder git-buildpackage
+cowbuilder --create --distribution buster --architecture amd64 --basepath /var/cache/pbuilder/base-$DIST-amd64.cow --mirror http://ftp.debian.org/debian/ --components=main
+
+# We need to make sure our build process can use networking in order to pip install the requirements
+echo "USENETWORK=yes" > ~/.pbuilderrc
+
+# Then clone the repository and build the .deb
+git clone https://github.com/ByteInternet/hypernode-docs-next
+cd hypernode-docs-next
+gbp buildpackage --git-pbuilder --git-dist=$DIST --git-arch=$ARCH --git-ignore-branch -us -uc -sa --git-ignore-new
+```
+
+Then after building the Deb you could install it with dpkg. For example:
+```
+dpkg -i ../hndocsnext_20230121.173551_all.deb
+```
+
+And test it out with:
+```
+man hypernode
+```
+
+To inspect the contents of the deb archive you can run:
+```
+# dpkg -L hndocsnext
+/.
+/usr
+/usr/local
+/usr/local/man
+/usr/local/man/man3
+/usr/local/man/man3/hypernode.3
+/usr/share
+/usr/share/doc
+/usr/share/doc/hndocsnext
+/usr/share/doc/hndocsnext/README.md
+/usr/share/doc/hndocsnext/changelog.gz
+```

bin/build_docs

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+set -e
+
 pushd docs
 make html
 popd

bin/build_manpage

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -e
+
+pushd docs
+make man
+echo "manpage located at $(realpath _build/man/docs.1)"
+popd

bin/find_dead_links

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+
+import multiprocessing
+import re
+from glob import glob
+from typing import List
+
+import requests
+
+from hypernode.common.settings import DOCS_DIR
+
+FAKE_DOMAINS = [
+    "example.com",
+    "yourdomain.com",
+    "example.hypernode.io",
+    "hypernode.local",
+    "127.0.0.1",
+]
+
+
+def get_links(md_file: str) -> List[str]:
+    with open(md_file) as f:
+        content = f.read()
+    return re.findall(r"\[.*\]\((http.+?)\)", content)
+
+
+def is_link_is_dead(link: str) -> bool:
+    try:
+        resp = requests.get(link, timeout=5)
+    except Exception as e:
+        print(f"Couldn't get {link}: {e}")
+        return True
+    dead = resp.status_code in [404, 500, 502, 503, 504]
+    if dead:
+        print(f"Dead link: {link}")
+    return dead
+
+
+def main():
+    links = []
+    for md_file in glob(f"{DOCS_DIR}/**/*.md", recursive=True):
+        for link in get_links(md_file):
+            if not any(fake_domain in link for fake_domain in FAKE_DOMAINS):
+                links.append(link)
+    links = list(set(links))
+    print(f"Found {len(links)} unique links")
+    # Loop over links in a multiprocessing pool
+    with multiprocessing.Pool(4) as p:
+        p.map(is_link_is_dead, links)
+
+
+if __name__ == "__main__":
+    main()

bin/generate_nginx_redirects

@@ -0,0 +1,5 @@
+#!/usr/bin/env python3
+from hypernode.redirect.generate_nginx_redirects import main
+
+if __name__ == "__main__":
+    main()

bin/import_metadata

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from hypernode.metadata.importer import main
+
+if __name__ == "__main__":
+    main()

ci/bin/detect_rewrites.py

@@ -0,0 +1,53 @@
+import argparse
+import re
+import subprocess
+from typing import Tuple
+
+RENAME_PATTERN = re.compile(r"^R[0-9]+")
+
+
+def parse_args() -> Tuple[str, str]:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("start_commit")
+    parser.add_argument("end_commit")
+
+    args = parser.parse_args()
+
+    return args.start_commit, args.end_commit
+
+
+def main():
+    start_commit, end_commit = parse_args()
+    diff_command = [
+        "git",
+        "diff",
+        "--name-status",
+        "-M",
+        "--stat",
+        start_commit,
+        end_commit,
+    ]
+    output = subprocess.check_output(diff_command, encoding="utf-8")
+    renamed_files = filter(RENAME_PATTERN.match, output.splitlines())
+    renamed_files = map(lambda x: x.split("\t")[1:], renamed_files)
+    renamed_markdown_files = filter(lambda x: x[0].endswith(".md"), renamed_files)
+    renamed_markdown_files = list(renamed_markdown_files)
+
+    if renamed_markdown_files:
+        print("Detected renamed/moved files:")
+        print("```")
+        for file_from, file_to in renamed_markdown_files:
+            print("{} => {}".format(file_from, file_to))
+        print("```")
+        print("To prevent 404 pages, consider adding the redirects to the new pages:")
+        print("```markdown")
+        print("---")
+        print("redirect_from:")
+        for file_from, _ in renamed_markdown_files:
+            print("  - {}".format(file_from))
+        print("---")
+        print("```")
+
+
+if __name__ == "__main__":
+    main()

debian/changelog

@@ -0,0 +1,6 @@
+hndocsnext (20230121.173551) UNRELEASED; urgency=medium
+
+  [ Rick van de Loo ]
+  * Initial packaging
+
+ -- Rick van de Loo <vdloo@desktop>  Sat, 21 Jan 2023 17:35:51 +0200

debian/compat

@@ -0,0 +1 @@
+9

debian/control

@@ -0,0 +1,13 @@
+Source: hndocsnext
+Priority: optional
+Maintainer: Hypernode Tech Team <[email protected]>
+Build-Depends: debhelper (>= 9), python3, python3-venv, python3-dev
+Standards-Version: 3.9.4
+Homepage: https://github.com/ByteInternet/hypernode-docs-next
+Vcs-Git: [email protected]:ByteInternet/hypernode-docs-next.git
+
+Package: hndocsnext
+Section: doc
+Architecture: all
+Depends: ${misc:Depends}
+Description: Hypernode docs in manpage form. Run 'man hypernode'.

debian/docs

@@ -0,0 +1 @@
+README.md

debian/hndocsnext.install

@@ -0,0 +1,3 @@
+# https://www.debian.org/doc/manuals/maint-guide/dother.en.html#install
+
+docs/_build/man/hypernode.3 /usr/local/man/man3/

debian/hndocsnext.postinst

@@ -0,0 +1,3 @@
+#!/bin/sh
+set -e
+mandb

debian/rules

@@ -0,0 +1,17 @@
+#!/usr/bin/make -f
+# -*- makefile -*-
+
+# Uncomment this to turn on verbose mode.
+#export DH_VERBOSE=1
+.PHONY: build
+
+build:
+	python3 -m venv venv
+	venv/bin/pip install -r requirements/development.txt
+	bash -c 'source venv/bin/activate; bin/build_manpage'
+	mv docs/_build/man/docs.1 docs/_build/man/hypernode.3
+
+override_dh_usrlocal:
+
+%:
+	dh $@

debian/source/format

@@ -0,0 +1 @@
+3.0 (native)