numa.c

/*
 * Copyright (c) 2022 The DragonFly Project.  All rights reserved.
 *
 * This code is derived from software contributed to The DragonFly Project
 * by Matthew Dillon <dillon@backplane.com>
 *
 * This code uses concepts and configuration based on 'synth', by
 * John R. Marino <draco@marino.st>, which was written in ada.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 * 3. Neither the name of The DragonFly Project nor the names of its
 *    contributors may be used to endorse or promote products derived
 *    from this software without specific, prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */
/*
 * Put builders into (numa) cpu domains to improve cache interactions
 * (This is totally optional)
 */
#include "dsynth.h"

/*
 * Number of NUMA domains or integral multiple of same.  Recommend a
 * value of 0 (disabled), 2 or 4.  Set with Numa_setsize parameter.
 */
int NumaSetSize = 0;

/*
 * Set the domain for the given slot number.  -1 == reset to default.
 */
void
setNumaDomain(int slot)
{
	static int numa_initialized;
	static cpu_set_t defset;

	/*
	 * Auto-initialization (0) -> Enabled (1) or Disabled (-1)
	 */
	if (numa_initialized == 0) {
		if (NumaSetSize <= 1) {
			/*
			 * Disabled
			 */
			numa_initialized = -1;
		} else if (pthread_getaffinity_np(pthread_self(),
					          sizeof(defset), &defset) != 0)
	        {
			/*
			 * Missing pthreads support
			 */
			dlog(DLOG_ALL,
			     "Warning: pthread_*affinity*() "
			     "not supported\n");
			numa_initialized = -1;
		} else {
			/*
			 * Operational, default set saved
			 */
			numa_initialized = 1;
		}
	}

	/*
	 * Do nothing if disabled
	 */
	if (numa_initialized < 0)
		return;

	/*
	 * Set domain for requested slot number or -1 to return to default
	 */
	if (slot < 0) {
		/*
		 * Return to default
		 */
		pthread_setaffinity_np(pthread_self(), sizeof(defset), &defset);
	} else if (NumCores <= NumaSetSize * 2) {
		/*
		 * Not enough cores to partition
		 */
		/* do nothing */
	} else {
		/*
		 * Set cpumask to N-way partitioned mask of cpu threads,
		 * hopefully in the same NUMA domain.
		 *
		 * XXX hacked for the moment to assume a simple partitioning
		 * in linear blocks of cores, with the second half of the
		 * cpu space being the related sibling hyperthreads.
		 */
		cpu_set_t cpuset;
		int hcores = NumCores / 2;
		int count = (hcores + NumaSetSize - 1) / NumaSetSize;
		int i;

		slot = slot % NumaSetSize;
		slot *= count;
		slot %= hcores;
		CPU_ZERO(&cpuset);
		for (i = 0; i < count; ++i) {
			CPU_SET(slot + i, &cpuset);
			CPU_SET(slot + hcores + i, &cpuset);
		}
		pthread_setaffinity_np(pthread_self(), sizeof(cpuset), &cpuset);
	}
}