Smaller tweaks from ipc/ to flow/ repo. #340

Workflow file for this run

	name: Flow pipeline

	# TODO: Code reuse somehow versus Flow-IPC workflow counterpart file? Could apply to major parts of this file.
	# The copy/pasting and "see comment in other file" types of comments are a bit unseemly.

	on:
	# Want to merge to development tip? Should probably pass these builds/tests/doc generation first.
	pull_request:
	branches:
	- main
	push:
	# Was able to merge to development tip? Should ensure these builds/tests/doc generation pass after the fact;
	# plus auto-check-in generated docs into development tip branch as well (and signal web site to update accordingly).
	branches:
	- main
	# Created release tag? Just in case, should ensure these builds/tests/doc generation pass after the fact still;
	# plus signal web site to update accordingly (new release/docs for it).
	tags:
	- v*
	# To create the button that runs a workflow manually (has to be in `main` at least):
	workflow_dispatch:

	jobs:
	# See comments in Flow-IPC workflow counterpart. We use same techniques.
	set-vars:
	runs-on: ubuntu-latest
	steps:
	- name: Set variables/constants for subsequent use
	run: \|
	# Set variables/constants for subsequent use.
	outputs:
	doc-commit-message: (Commit by workflow script) Update generated documentation.

	# Impetus behind this is to gate whether the real jobs below actually need to run. Can of course also compute
	# other things as needed.
	setup:
	needs: set-vars
	runs-on: ubuntu-latest
	steps:
	- id: compute_proceed_else_not
	name: Compute whether for main jobs to proceed or not
	# See comments in Flow-IPC workflow counterpart. We use same techniques.
	run: \|
	# Compute whether for main jobs to proceed or not.
	TMP_MSG=/tmp/flow-pipeline-head-cmt-msg.txt
	cat <<'FLOW_PIPELINE_HEAD_CMD_MSG_EOF' > $TMP_MSG
	${{ github.event.head_commit.message }}
	FLOW_PIPELINE_HEAD_CMD_MSG_EOF
	if [ '${{ github.ref }}' != 'refs/heads/main' ] \|\| \
	[ '${{ github.event_name }}' != 'push' ] \|\| \
	! { head --lines=1 $TMP_MSG \| grep -Fxq '${{ needs.set-vars.outputs.doc-commit-message }}'; }; then
	echo 'proceed-else-not=true' >> $GITHUB_OUTPUT
	else
	echo 'proceed-else-not=false' >> $GITHUB_OUTPUT
	echo 'The real jobs will not run: earlier `doc` job checked-in generated docs to `main`.'
	echo 'That is not a source change and requires no actual pipeline to execute.'
	fi
	outputs:
	proceed-else-not: ${{ steps.compute_proceed_else_not.outputs.proceed-else-not }}

	build:
	needs: [setup, set-vars]
	if: \|
	needs.setup.outputs.proceed-else-not == 'true'

	strategy:
	fail-fast: false
	matrix:
	compiler:
	- id: gcc-9
	name: gcc
	version: 9
	c-path: /usr/bin/gcc-9
	cpp-path: /usr/bin/g++-9
	- id: gcc-10
	name: gcc
	version: 10
	c-path: /usr/bin/gcc-10
	cpp-path: /usr/bin/g++-10
	- id: gcc-11
	name: gcc
	version: 11
	c-path: /usr/bin/gcc-11
	cpp-path: /usr/bin/g++-11
	- id: gcc-13
	name: gcc
	version: 13
	c-path: /usr/bin/gcc-13
	cpp-path: /usr/bin/g++-13
	- id: clang-13
	name: clang
	version: 13
	c-path: /usr/bin/clang-13
	cpp-path: /usr/bin/clang++-13
	- id: clang-15
	name: clang
	version: 15
	c-path: /usr/bin/clang-15
	cpp-path: /usr/bin/clang++-15
	- id: clang-16
	name: clang
	version: 16
	c-path: /usr/bin/clang-16
	cpp-path: /usr/bin/clang++-16
	install: True
	- id: clang-17
	name: clang
	version: 17
	c-path: /usr/bin/clang-17
	cpp-path: /usr/bin/clang++-17
	install: True
	build-test-cfg:
	- id: debug
	conan-profile-build-type: Debug
	- id: release
	conan-profile-build-type: Release
	- id: relwithdebinfo
	conan-profile-build-type: RelWithDebInfo
	- id: minsizerel
	conan-profile-build-type: MinSizeRel
	# The sanitized (-*san) build/test configs follow. Here we follow Flow-IPC workflow counterpart.
	# This can be a non-trivial topic -- though by definition Flow's situation is simpler than Flow-IPC's,
	# as the latter includes the former entirely -- so if changing or trying to understand sanitizer-related
	# stuff, please check the Flow-IPC workflow counterpart file, particularly comments near each thing.
	# Keeping comments here light.
	#
	# Note that, as of this writing, MSAN support is present in Flow-IPC workflow counterpart but is disabled
	# for reasons explained therein. We therefore omit it here entirely; no need to have it taking up space here
	# too.
	- id: relwithdebinfo-asan
	conan-profile-build-type: RelWithDebInfo
	conan-profile-custom-conf: \|
	tools.build:cflags = ["-fsanitize=address", "-fno-omit-frame-pointer"]
	tools.build:cxxflags = ["-fsanitize=address", "-fno-omit-frame-pointer"]
	tools.build:sharedlinkflags = ["-fsanitize=address"]
	tools.build:exelinkflags = ["-fsanitize=address"]
	conan-profile-custom-settings: \|
	compiler.sanitizer = address
	conan-custom-settings-defs: \| # Could we not copy/paste these 4x?
	data['compiler']['gcc']['sanitizer'] = ['None', 'address', 'thread', 'memory', 'undefined']
	data['compiler']['clang']['sanitizer'] = ['None', 'address', 'thread', 'memory', 'undefined']
	sanitizer-name: asan
	no-lto: True
	- id: relwithdebinfo-ubsan
	conan-profile-build-type: RelWithDebInfo
	conan-profile-custom-conf: \|
	tools.build:cflags = ["-fsanitize=undefined", "-fno-omit-frame-pointer"]
	tools.build:cxxflags = ["-fsanitize=undefined", "-fno-omit-frame-pointer"]
	tools.build:sharedlinkflags = ["-fsanitize=undefined"]
	tools.build:exelinkflags = ["-fsanitize=undefined"]
	conan-profile-custom-settings: \|
	compiler.sanitizer = undefined
	conan-custom-settings-defs: \|
	data['compiler']['gcc']['sanitizer'] = ['None', 'address', 'thread', 'memory', 'undefined']
	data['compiler']['clang']['sanitizer'] = ['None', 'address', 'thread', 'memory', 'undefined']
	sanitizer-name: ubsan
	no-lto: True
	- id: relwithdebinfo-tsan
	conan-profile-build-type: RelWithDebInfo
	conan-profile-custom-conf: \|
	tools.build:cflags = ["-fsanitize=thread"]
	tools.build:cxxflags = ["-fsanitize=thread"]
	tools.build:sharedlinkflags = ["-fsanitize=thread"]
	tools.build:exelinkflags = ["-fsanitize=thread"]
	conan-profile-custom-settings: \|
	compiler.sanitizer = thread
	conan-custom-settings-defs: \|
	data['compiler']['gcc']['sanitizer'] = ['None', 'address', 'thread', 'memory', 'undefined']
	data['compiler']['clang']['sanitizer'] = ['None', 'address', 'thread', 'memory', 'undefined']
	sanitizer-name: tsan
	no-lto: True

	# Again, these exclusions are explained in FLow-IPC workflow counterpart.
	exclude:
	- compiler: { id: gcc-9 }
	build-test-cfg: { id: relwithdebinfo-asan }
	- compiler: { id: gcc-10 }
	build-test-cfg: { id: relwithdebinfo-asan }
	- compiler: { id: gcc-11 }
	build-test-cfg: { id: relwithdebinfo-asan }
	- compiler: { id: gcc-13 }
	build-test-cfg: { id: relwithdebinfo-asan }
	- compiler: { id: clang-13 }
	build-test-cfg: { id: relwithdebinfo-asan }
	- compiler: { id: gcc-9 }
	build-test-cfg: { id: relwithdebinfo-ubsan }
	- compiler: { id: gcc-10 }
	build-test-cfg: { id: relwithdebinfo-ubsan }
	- compiler: { id: gcc-11 }
	build-test-cfg: { id: relwithdebinfo-ubsan }
	- compiler: { id: gcc-13 }
	build-test-cfg: { id: relwithdebinfo-ubsan }
	- compiler: { id: clang-13 }
	build-test-cfg: { id: relwithdebinfo-ubsan }
	- compiler: { id: gcc-9 }
	build-test-cfg: { id: relwithdebinfo-tsan }
	- compiler: { id: gcc-10 }
	build-test-cfg: { id: relwithdebinfo-tsan }
	- compiler: { id: gcc-11 }
	build-test-cfg: { id: relwithdebinfo-tsan }
	- compiler: { id: gcc-13 }
	build-test-cfg: { id: relwithdebinfo-tsan }
	- compiler: { id: clang-13 }
	build-test-cfg: { id: relwithdebinfo-tsan }

	# Not using ubuntu-latest, so as to avoid surprises with OS upgrades and such.
	runs-on: ubuntu-22.04

	name: ${{ matrix.compiler.id }}-${{ matrix.build-test-cfg.id }}

	env:
	build-dir: ${{ github.workspace }}/build/${{ matrix.build-test-cfg.conan-profile-build-type }}
	install-root-dir: ${{ github.workspace }}/install/${{ matrix.build-test-cfg.conan-profile-build-type }}
	# (Unfortunately cannot refer to earlier-assigned `env.` entries within subsequent ones.)
	install-dir: ${{ github.workspace }}/install/${{ matrix.build-test-cfg.conan-profile-build-type }}/usr/local
	# For the remaining env entries please see comments in Flow-IPC workflow counterpart. We use same techniques.
	san-suppress-cfg-file: ${{ github.workspace }}/install/${{ matrix.build-test-cfg.conan-profile-build-type }}/usr/local/bin/san_suppressions.cfg
	san-suppress-cfg-in-file1: sanitize/${{ matrix.build-test-cfg.sanitizer-name }}/suppressions_${{ matrix.compiler.name }}.cfg
	san-suppress-cfg-in-file2: sanitize/${{ matrix.build-test-cfg.sanitizer-name }}/suppressions_${{ matrix.compiler.name }}_${{ matrix.compiler.version }}.cfg
	setup-tests-env: \|
	if [ '${{ matrix.build-test-cfg.sanitizer-name }}' = asan ]; then
	export ASAN_OPTIONS='disable_coredump=0'
	echo "ASAN_OPTIONS = [$ASAN_OPTIONS]."
	elif [ '${{ matrix.build-test-cfg.sanitizer-name }}' = ubsan ]; then
	export SAN_SUPP=1
	export SAN_SUPP_CFG=${{ github.workspace }}/install/${{ matrix.build-test-cfg.conan-profile-build-type }}/usr/local/bin/san_suppressions.cfg
	export UBSAN_OPTIONS="disable_coredump=0 print_stacktrace=1 suppressions=$SAN_SUPP_CFG"
	echo "UBSAN_OPTIONS = [$UBSAN_OPTIONS]."
	elif [ '${{ matrix.build-test-cfg.sanitizer-name }}' = tsan ]; then
	export SAN_SUPP=1
	export SAN_SUPP_CFG=${{ github.workspace }}/install/${{ matrix.build-test-cfg.conan-profile-build-type }}/usr/local/bin/san_suppressions.cfg
	export TSAN_OPTIONS="disable_coredump=0 second_deadlock_stack=1 suppressions=$SAN_SUPP_CFG"
	echo "TSAN_OPTIONS = [$TSAN_OPTIONS]."
	fi
	# TODO: Add more *SAN here.
	if [ "$SAN_SUPP" != '' ]; then
	echo 'Sanitizer [${{ matrix.build-test-cfg.sanitizer-name }}] suppressions cfg contents:'
	echo '[[[ file--'
	cat $SAN_SUPP_CFG
	echo '--file ]]]'
	fi

	steps:
	- name: Checkout `flow` repository
	uses: actions/checkout@v4

	- name: Update available software list for apt-get
	run: sudo apt-get update

	- name: Install clang compiler
	if: \|
	matrix.compiler.install && (matrix.compiler.name == 'clang')
	run: \|
	# Install clang compiler.
	wget https://apt.llvm.org/llvm.sh
	chmod u+x llvm.sh
	sudo ./llvm.sh ${{ matrix.compiler.version }}

	- name: Install gcc compiler
	if: \|
	matrix.compiler.install && (matrix.compiler.name == 'gcc')
	run: \|
	# Install gcc compiler.
	sudo apt-get install -y software-properties-common
	sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y
	sudo apt-get update
	sudo apt-get install -y gcc-${{ matrix.compiler.version }} g++-${{ matrix.compiler.version }}

	# We get highest 1.x, instead of 2+, essentially to stay even with the Flow-IPC pipeline (and its similar workflow
	# file). See explanation in that file. The to-do there applies to us as well (both should be done
	# at ~the same time).
	- name: Install the latest version of Conan which is less than 2
	run: pip install 'conan<2'

	- name: Add custom settings for Conan packages
	if: \|
	matrix.build-test-cfg.conan-custom-settings-defs
	run: \|
	# Add custom settings for Conan packages.
	conan config init
	pip install PyYAML
	CONAN_SETTINGS_PATH=$(conan config home)/settings.yml
	python -c "

	import yaml

	with open('$CONAN_SETTINGS_PATH', 'r') as file:
	data = yaml.safe_load(file)

	${{ matrix.build-test-cfg.conan-custom-settings-defs }}

	with open('$CONAN_SETTINGS_PATH', 'w') as file:
	yaml.dump(data, file)
	"

	- name: Create Conan profile
	run: \|
	# Create Conan profile.
	cat <<'EOF' > conan_profile
	[settings]
	compiler = ${{ matrix.compiler.name }}
	compiler.version = ${{ matrix.compiler.version }}
	compiler.cppstd = 17
	# TODO: Consider testing with LLVM-libc++ also (with clang anyway).
	compiler.libcxx = libstdc++11
	arch = x86_64
	os = Linux
	build_type = ${{ matrix.build-test-cfg.conan-profile-build-type }}
	${{ matrix.build-test-cfg.conan-profile-custom-settings }}

	[conf]
	tools.build:compiler_executables = {"c": "${{ matrix.compiler.c-path }}", "cpp": "${{ matrix.compiler.cpp-path }}"}
	tools.env.virtualenv:auto_use = True
	${{ matrix.build-test-cfg.conan-profile-custom-conf }}

	[buildenv]
	CC = ${{ matrix.compiler.c-path }}
	CXX = ${{ matrix.compiler.cpp-path }}

	[options]
	flow:build = True
	flow:doc = False
	EOF

	- name: Install Flow dependencies with Conan using the profile
	run: conan install . --profile:build conan_profile --profile:host conan_profile --build missing

	- name: Build library and demos/tests with Conan
	run: conan build .

	- name: Install built targets with Makefile
	run: \|
	make install \
	--directory ${{ env.build-dir }} DESTDIR=${{ env.install-root-dir }}
	# Save runner space: blow away build dir after install.
	rm -rf ${{ env.build-dir }}

	# From now on use !cancelled() to try to run any test/demo that exists regardless
	# of preceding failures if any. Same-ish (always()) with the log-upload at the end.
	# Worst-case they'll all fail in super-basic ways and immediately; no extra harm done.

	# From now on save logs in install-dir/bin/logs. They will be tarred up and uploaded
	# as artifacts at the end. Please see comments in Flow-IPC workflow counterpart.
	# We use the same techniques and reasoning. Those may seem overkill given the small number
	# of tests/demos below and their small amount of output (as of this writing); but this can
	# and will grow. The techniques will still apply.

	- name: Run test/demo [NetFlow echo]
	run: \|
	# Run test/demo [NetFlow echo].
	cd ${{ env.install-dir }}/bin
	OUT_DIR=logs/net_flow_echo
	mkdir -p $OUT_DIR
	SUPP_DIR_A=${{ github.workspace }}/src
	{ cat $SUPP_DIR_A/${{ env.san-suppress-cfg-in-file1 }} $SUPP_DIR_A/${{ env.san-suppress-cfg-in-file2 }} \
	> ${{ env.san-suppress-cfg-file }} 2> /dev/null; } \|\| true
	${{ env.setup-tests-env }}
	./net_flow_echo_srv.exec 8888 localhost > $OUT_DIR/srv.console.log 2>&1 &
	sleep 1
	if ./net_flow_echo_cli.exec 1 127.0.01 8888 'Hello world!' > $OUT_DIR/cli.console.log 2>&1; \
	then EC=0; else EC=$?; fi
	echo "Client finished with code [$EC]."
	echo 'Server may print exception due to SIGTERM interruption; that is fine.'
	kill `pidof $PWD/net_flow_echo_srv.exec`
	# The executables also produce Flow logs with these hard-coded names in CWD.
	touch flow_echo_srv.log # Create if needed just in case.
	touch flow_echo_cli.log
	mv flow_echo_srv.log $OUT_DIR/srv.log
	mv flow_echo_cli.log $OUT_DIR/cli.log
	[ $EC -eq 0 ]

	# (More tests/demos here as needed.)

	- name: Check test/demo logs for non-fatal sanitizer error(s)
	if: \|
	(!cancelled()) && (matrix.build-test-cfg.sanitizer-name == 'ubsan')
	run: \|
	# Check test/demo logs for non-fatal sanitizer error(s)
	cd ${{ env.install-dir }}/bin/logs
	# grep returns 0 if 1+ found, 1 if none found, 2+ on error. So check results explicitly instead of -e.
	# Namely, for us, the only OK result is an empty stdout and empty stderr.
	# Otherwise either grep failed (unlikely) or found 1+ problems; either way redirection target will
	# be not-empty, and we will force failure.
	{ grep 'SUMMARY: UndefinedBehaviorSanitizer:' `find . -type f` > san_failure_summaries.txt 2>&1; } \|\| true
	if [ -s san_failure_summaries.txt ]; then
	echo 'Error(s) found. Pipeline will fail. Failures summarized below.'
	echo 'Please peruse uploaded log artifacts, resolve the issues, and run pipeline again.'
	echo '[[[ file--'
	cat san_failure_summaries.txt
	echo '--file ]]]'
	false
	fi
	echo 'No errors found in logs.'

	- name: Package test/demo logs tarball
	if: \|
	always()
	run: \|
	# Package test/demo logs tarball.
	cd ${{ env.install-dir }}/bin
	tar cvzf logs.tgz logs
	rm -rf logs # Save runner space.

	- name: Upload test/demo logs (please inspect if failure(s) seen above)
	if: \|
	always()
	uses: actions/upload-artifact@v3
	with:
	name: flow-test-logs-${{ matrix.compiler.id }}-${{ matrix.build-test-cfg.id }}
	path: ${{ env.install-dir }}/bin/logs.tgz

	doc:
	needs: [setup, set-vars]
	if: \|
	needs.setup.outputs.proceed-else-not == 'true'

	strategy:
	fail-fast: false
	matrix:
	compiler:
	# Pick a reasonably modern but pre-installed compiler for building Doxygen/etc.
	- id: clang-15
	name: clang
	version: 15
	c-path: /usr/bin/clang-15
	cpp-path: /usr/bin/clang++-15
	build-cfg:
	- id: release
	conan-profile-build-type: Release
	conan-preset: release

	runs-on: ubuntu-22.04

	name: doc-${{ matrix.compiler.id }}-${{ matrix.build-cfg.id }}

	steps:
	- name: Update available software list for apt-get
	run: sudo apt-get update

	- name: Checkout `flow` repository
	uses: actions/checkout@v4

	- name: Install Flow dependencies (like Graphviz) with apt-get
	run: sudo apt-get install -y graphviz

	- name: Install the latest version of Conan which is less than 2
	run: pip install 'conan<2'

	- name: Create Conan profile
	run: \|
	# Create Conan profile.
	cat <<EOF > conan_profile
	[settings]
	compiler = ${{ matrix.compiler.name }}
	compiler.version = ${{ matrix.compiler.version }}
	compiler.cppstd = 17
	compiler.libcxx = libstdc++11
	arch = x86_64
	os = Linux
	build_type = ${{ matrix.build-cfg.conan-profile-build-type }}

	[conf]
	tools.build:compiler_executables = { "c": "${{ matrix.compiler.c-path }}", "cpp": "${{ matrix.compiler.cpp-path }}" }
	tools.env.virtualenv:auto_use = True

	[buildenv]
	CC = ${{ matrix.compiler.c-path }}
	CXX = ${{ matrix.compiler.cpp-path }}

	[options]
	flow:build = False
	flow:doc = True
	EOF

	- name: Install Flow dependencies (like Doxygen) with Conan using the profile
	run: conan install . --profile:build conan_profile --profile:host conan_profile --build missing

	- name: Generate code documentation using Conan and Doxygen
	run: conan build .

	- name: Create documentation tarball (full docs, API-only docs, landing page)
	run: \|
	# Create documentation tarball (full docs, API-only docs, landing page).
	cd ${{ github.workspace }}/doc/flow_doc
	${{ github.workspace }}/tools/doc/stage_generated_docs.sh \
	${{ github.workspace }}/build/${{ matrix.build-cfg.conan-profile-build-type }}

	- name: Upload documentation tarball
	uses: actions/upload-artifact@v3
	with:
	name: flow-doc
	path: ${{ github.workspace }}/doc/flow_doc.tgz

	# See comments in Flow-IPC workflow counterpart. We use same techniques.
	- name: (`main` branch only) Check-in generated documentation directly into source control
	id: doc_check_in
	if: success() && (!cancelled()) && (github.ref == 'refs/heads/main')
	run: \|
	# (`main` branch only) Check-in generated documentation directly into source control.
	echo 'generated/ docs have been added or replaced locally; mirroring this into checked-in tree.'
	git config user.name github-actions
	git config user.email [email protected]
	git config --local http.https://github.com/.extraheader \
	"AUTHORIZATION: basic $(echo -n x-access-token:${{ secrets.GIT_BOT_PAT }} \| base64)"
	cd ${{ github.workspace }}/doc/flow_doc
	git rm -r --cached generated \|\| echo 'No generated/ currently checked in; no problem.'
	git add generated
	git commit -m '${{ needs.set-vars.outputs.doc-commit-message }}'
	git push origin main

	# See comments in Flow-IPC workflow counterpart. We use same techniques.
	- name: (`main` branch only) Signal Pages that we have updated the generated docs (web site should update)
	if: success() && (!cancelled()) && (steps.doc_check_in.outcome != 'skipped')
	run: \|
	# Signal Pages that we have updated the generated docs (web site should update).
	curl --fail-with-body -X POST \
	-H 'Accept: application/vnd.github.v3+json' \
	-H 'Authorization: token ${{ secrets.GIT_BOT_PAT }}' \
	'https://api.github.com/repos/Flow-IPC/flow-ipc.github.io/dispatches' \
	-d '{"event_type": "flow-sync-doc-event"}'

	# See comments in Flow-IPC workflow counterpart. We use same techniques.
	- name: Signal Pages if this is running due to release creation (then web site should update)
	if: success() && (!cancelled()) && startsWith(github.ref, 'refs/tags/v')
	run: \|
	# Signal Pages if this is running due to release creation (then web site should update).
	curl --fail-with-body -X POST \
	-H 'Accept: application/vnd.github.v3+json' \
	-H 'Authorization: token ${{ secrets.GIT_BOT_PAT }}' \
	'https://api.github.com/repos/Flow-IPC/flow-ipc.github.io/dispatches' \
	-d '{"event_type": "flow-sync-doc-event", "client_payload": {"version": "${{ github.ref_name }}"}}'

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Smaller tweaks from ipc/ to flow/ repo. #340

Workflow file

Smaller tweaks from ipc/ to flow/ repo. #340

Jobs

Run details

Workflow file for this run