Initial Commit

2gb.patch

@@ -0,0 +1,11 @@
+--- squashfs-tools/mksquashfs.c.orig	2014-09-13 11:08:27.352318167 -0500
++++ squashfs-tools/mksquashfs.c	2014-09-13 11:09:36.701132044 -0500
+@@ -2055,7 +2055,7 @@
+
+ inline int is_fragment(struct inode_info *inode)
+ {
+-	int file_size = inode->buf.st_size;
++	off_t file_size = inode->buf.st_size;
+
+ 	/*
+ 	 * If this block is to be compressed differently to the

PAE.patch

@@ -0,0 +1,159 @@
+From 55f7ba830d40d438f0b0663a505e0c227fc68b6b Mon Sep 17 00:00:00 2001
+From: Phillip Lougher <[email protected]>
+Date: Tue, 10 Jun 2014 21:51:52 +0100
+Subject: mksquashfs: fix phys mem calculation for 32-bit processes on
+ PAE/64-bit kernels
+
+When adding the code to base default memory usage on physical memory
+(by default use 25% of physical memory), I made an oversight.  I assumed
+the process would be able to address 25% of physical memory.
+
+However, for 32-bit processes running on a PAE kernel or 64-bit kernel,
+25% of physical memory can easily exceed the addressible memory for a
+32-bit process, e.g. if a machine has 24 GB of physical memory, the
+code would asume the process could easily use 6 GB.
+
+A 32-bit process by definition can only address 4 GB (32-bit pointers).
+But, due to the typical kernel/user-space split (1GB/3GB, or 2GB/2GB)
+on PAE kernels, a 32-bit process may only be able to address 2 GB.
+
+So, if Mksquashfs is a 32-bit application running on a PAE/64-bit kernel,
+the code assumes it can address much more memory than it really can, which
+means it runs out of memory.
+
+The fix is to impose a maximum default limit on 32-bit kernels, or
+otherwise to never use a value more than 25% of the address space.  If
+we assume the maximum address space is 2 GB, then the maximum becomes
+512 MB.  But, given most kernels used the 1GB/3GB split, that may be
+unduely conservative, and 25% of 3 GB (756 MB) may be better.  This
+patch compromises on 640 MB, which is mid-way between the 512 MB and 756 MB
+values.  It is also the fixed default value previously used by Mksquashfs.
+
+This patch also alters the code which imposes a maximum size.  Previously
+it was believed limiting to the physical memory size was adequate.  But
+obviously this needs to be updated to take into account a 32-bit process
+may only be able to address 2 GB.  In the process I've also taken the
+opportunity to limit all requests to no more than 75% of physical memory.
+
+Signed-off-by: Phillip Lougher <[email protected]>
+
+diff --git a/squashfs-tools/mksquashfs.c b/squashfs-tools/mksquashfs.c
+index 86f82bb..5370ecf 100644
+--- a/squashfs-tools/mksquashfs.c
++++ b/squashfs-tools/mksquashfs.c
+@@ -304,7 +304,7 @@ void restorefs();
+ struct dir_info *scan1_opendir(char *pathname, char *subpath, int depth);
+ void write_filesystem_tables(struct squashfs_super_block *sBlk, int nopad);
+ unsigned short get_checksum_mem(char *buff, int bytes);
+-int get_physical_memory();
++void check_usable_phys_mem(int total_mem);
+
+
+ void prep_exit()
+@@ -4053,11 +4053,7 @@ void initialise_threads(int readq, int fragq, int bwriteq, int fwriteq,
+ 		BAD_ERROR("Queue sizes rediculously too large\n");
+ 	total_mem += fwriteq;
+
+-	if(total_mem > get_physical_memory()) {
+-		ERROR("Total queue sizes larger than physical memory.\n");
+-		ERROR("Mksquashfs will exhaust physical memory and thrash.\n");
+-		BAD_ERROR("Queues too large\n");
+-	}
++	check_usable_phys_mem(total_mem);
+
+ 	/*
+ 	 * convert from queue size in Mbytes to queue size in
+@@ -4879,6 +4875,72 @@ int get_physical_memory()
+ }
+
+
++void check_usable_phys_mem(int total_mem)
++{
++	/*
++	 * We want to allow users to use as much of their physical
++	 * memory as they wish.  However, for practical reasons there are
++	 * limits which need to be imposed, to protect users from themselves
++	 * and to prevent people from using Mksquashfs as a DOS attack by using
++	 * all physical memory.   Mksquashfs uses memory to cache data from disk
++	 * to optimise performance.  It is pointless to ask it to use more
++	 * than 75% of physical memory, as this causes thrashing and it is thus
++	 * self-defeating.
++	 */
++	int mem = get_physical_memory();
++
++	mem = (mem >> 1) + (mem >> 2); /* 75% */
++						
++	if(total_mem > mem) {
++		ERROR("Total memory requested is more than 75%% of physical "
++						"memory.\n");
++		ERROR("Mksquashfs uses memory to cache data from disk to "
++						"optimise performance.\n");
++		ERROR("It is pointless to ask it to use more than this amount "
++						"of memory, as this\n");
++		ERROR("causes thrashing and it is thus self-defeating.\n");
++		BAD_ERROR("Requested memory size too large\n");
++	}
++
++	if(sizeof(void *) == 4 && total_mem > 2048) {
++		/*
++		 * If we're running on a kernel with PAE or on a 64-bit kernel,
++		 * then the 75% physical memory limit can still easily exceed
++		 * the addressable memory by this process.
++		 *
++		 * Due to the typical kernel/user-space split (1GB/3GB, or
++		 * 2GB/2GB), we have to conservatively assume the 32-bit
++		 * processes can only address 2-3GB.  So refuse if the user
++		 * tries to allocate more than 2GB.
++		 */
++		ERROR("Total memory requested may exceed maximum "
++				"addressable memory by this process\n");
++		BAD_ERROR("Requested memory size too large\n");
++	}
++}
++
++
++int get_default_phys_mem()
++{
++	int mem = get_physical_memory() / SQUASHFS_TAKE;
++
++	if(sizeof(void *) == 4 && mem > 640) {
++		/*
++		 * If we're running on a kernel with PAE or on a 64-bit kernel,
++		 * the default memory usage can exceed the addressable
++		 * memory by this process.
++		 * Due to the typical kernel/user-space split (1GB/3GB, or
++		 * 2GB/2GB), we have to conservatively assume the 32-bit
++		 * processes can only address 2-3GB.  So limit the  default
++		 * usage to 640M, which gives room for other data.
++		 */
++		mem = 640;
++	}
++
++	return mem;
++}
++
++
+ void calculate_queue_sizes(int mem, int *readq, int *fragq, int *bwriteq,
+ 							int *fwriteq)
+ {
+@@ -4890,7 +4952,7 @@ void calculate_queue_sizes(int mem, int *readq, int *fragq, int *bwriteq,
+
+
+ #define VERSION() \
+-	printf("mksquashfs version 4.3 (2014/05/12)\n");\
++	printf("mksquashfs version 4.3-git (2014/06/09)\n");\
+ 	printf("copyright (C) 2014 Phillip Lougher "\
+ 		"<[email protected]>\n\n"); \
+ 	printf("This program is free software; you can redistribute it and/or"\
+@@ -4918,7 +4980,7 @@ int main(int argc, char *argv[])
+ 	int fragq;
+ 	int bwriteq;
+ 	int fwriteq;
+-	int total_mem = get_physical_memory() / SQUASHFS_TAKE;
++	int total_mem = get_default_phys_mem();
+ 	int progress = TRUE;
+ 	int force_progress = FALSE;
+ 	struct file_buffer **fragment = NULL;
+-- 
+cgit v0.10.1
+

README.md

@@ -0,0 +1,12 @@
+# squashfs-tools EL 8 RPM Packaging
+
+This is the upstream EL8 squashfs-tools 4.3-19.el8 patched with:
+
+https://github.com/plougher/squashfs-tools/commit/de03266983ceb62e5365aac84fcd3b2fd4d16e6f
+
+To fix:
+
+https://bugzilla.redhat.com/show_bug.cgi?id=1785652
+
+It is needed to avoid sporadic rare failures of `mksquashfs` in Singularity testing on large multi-core machines where a very large number of `mksquashfs` runs take place during `make e2e-test`.
+

bz1602698.patch

@@ -0,0 +1,95 @@
+diff -Nupr a/squashfs-tools/action.c b/squashfs-tools/action.c
+--- a/squashfs-tools/action.c	2014-05-09 23:54:13.000000000 -0500
++++ b/squashfs-tools/action.c	2019-04-18 10:59:53.140496887 -0500
+@@ -1094,8 +1094,14 @@ static int parse_sym_mode_args(struct ac
+ 	for (i = 0; i < args; i++) {
+ 		struct mode_data *entry = parse_sym_mode_arg(argv[i]);
+
+-		if (entry == NULL)
++		if (entry == NULL) {
++			while (head) {
++				cur = head;
++				head = head->next;
++				free(cur);
++			}
+ 			return 0;
++		}
+
+ 		if (cur) {
+ 			cur->next = entry;
+diff -Nupr a/squashfs-tools/mksquashfs.c b/squashfs-tools/mksquashfs.c
+--- a/squashfs-tools/mksquashfs.c	2019-04-18 10:51:05.975460126 -0500
++++ b/squashfs-tools/mksquashfs.c	2019-04-18 11:04:16.860682497 -0500
+@@ -3537,11 +3537,11 @@ void dir_scan2(struct dir_info *dir, str
+ 			char *subpath = strdup(subpathname(dir_ent));
+ 			struct dir_info *sub_dir = scan1_opendir("", subpath,
+ 						dir->depth + 1);
++			free(subpath);
+ 			if(sub_dir == NULL) {
+ 				ERROR_START("Could not create pseudo directory "
+ 					"\"%s\"", pseudo_ent->pathname);
+ 				ERROR_EXIT(", skipping...\n");
+-				free(subpath);
+ 				pseudo_ino --;
+ 				continue;
+ 			}
+diff -Nupr a/squashfs-tools/read_fs.c b/squashfs-tools/read_fs.c
+--- a/squashfs-tools/read_fs.c	2014-05-09 23:54:13.000000000 -0500
++++ b/squashfs-tools/read_fs.c	2019-04-18 11:06:32.499233451 -0500
+@@ -974,7 +974,8 @@ long long read_filesystem(char *root_nam
+
+ error:
+ 	free(id_table);
+-	free(inode_table);
++	if (inode_table)
++		free(inode_table);
+ 	free(directory_table);
+ 	return 0;
+ }
+diff -Nupr a/squashfs-tools/unsquash-4.c b/squashfs-tools/unsquash-4.c
+--- a/squashfs-tools/unsquash-4.c	2019-04-18 10:51:05.972460124 -0500
++++ b/squashfs-tools/unsquash-4.c	2019-04-18 11:32:54.600160166 -0500
+@@ -50,9 +50,11 @@ int read_fragment_table_4(long long *dir
+ 			"fragment table index\n");
+
+ 	fragment_table = malloc(bytes);
+-	if(fragment_table == NULL)
++	if(fragment_table == NULL) {
++		free(fragment_table_index);
+ 		EXIT_UNSQUASH("read_fragment_table: failed to allocate "
+ 			"fragment table\n");
++	}
+
+ 	res = read_fs_bytes(fd, sBlk.s.fragment_table_start,
+ 		SQUASHFS_FRAGMENT_INDEX_BYTES(sBlk.s.fragments),
+@@ -60,7 +62,7 @@ int read_fragment_table_4(long long *dir
+ 	if(res == FALSE) {
+ 		ERROR("read_fragment_table: failed to read fragment table "
+ 			"index\n");
+-		return FALSE;
++		goto out;
+ 	}
+ 	SQUASHFS_INSWAP_FRAGMENT_INDEXES(fragment_table_index, indexes);
+
+@@ -75,7 +77,8 @@ int read_fragment_table_4(long long *dir
+ 		if(length == FALSE) {
+ 			ERROR("read_fragment_table: failed to read fragment "
+ 				"table index\n");
+-			return FALSE;
++			res = FALSE;
++			goto out;
+ 		}
+ 	}
+
+@@ -83,7 +86,10 @@ int read_fragment_table_4(long long *dir
+ 		SQUASHFS_INSWAP_FRAGMENT_ENTRY(&fragment_table[i]);
+
+ 	*directory_table_end = fragment_table_index[0];
+-	return TRUE;
++	res = TRUE;
++out:
++	free(fragment_table_index);
++	return res;
+ }
+
+

bz1624173.patch

@@ -0,0 +1,11 @@
+--- a/squashfs-tools/Makefile	2019-07-24 13:52:09.552920269 -0400
++++ b/squashfs-tools/Makefile	2019-07-24 23:52:47.839665687 -0400
+@@ -107,6 +107,8 @@ XATTR_DEFAULT = 1
+ #        End of BUILD options section         #
+ ###############################################
+
++EXTRA_LDFLAGS += -z now -pie
++
+ INCLUDEDIR = -I.
+ INSTALL_DIR = /usr/local/bin
+

bz1716278.patch

@@ -0,0 +1,21 @@
+--- a/squashfs-tools/mksquashfs.c	2019-08-06 11:03:34.071402614 -0400
++++ b/squashfs-tools/mksquashfs.c	2019-08-06 11:09:54.063340318 -0400
+@@ -267,6 +267,7 @@ pthread_mutex_t	pos_mutex = PTHREAD_MUTE
+ pthread_mutex_t	dup_mutex = PTHREAD_MUTEX_INITIALIZER;
+
+ /* user options that control parallelisation */
++#define MAX_CPUS 256
+ int processors = -1;
+ int bwriter_size;
+
+@@ -4124,6 +4125,10 @@ void initialise_threads(int readq, int f
+ #endif
+ 	}
+
++	/* Limit the number of processors we're going to use */
++	if(processors > MAX_CPUS)
++		processors = MAX_CPUS;
++
+ 	if(multiply_overflow(processors, 3) ||
+ 			multiply_overflow(processors * 3, sizeof(pthread_t)))
+ 		BAD_ERROR("Processors too large\n");

cve-2015-4645.patch

@@ -0,0 +1,29 @@
+diff --git a/squashfs-tools/unsquash-4.c b/squashfs-tools/unsquash-4.c
+index ecdaac796f09..2c0cf63daf67 100644
+--- a/squashfs-tools/unsquash-4.c
++++ b/squashfs-tools/unsquash-4.c
+@@ -31,9 +31,9 @@ static unsigned int *id_table;
+ int read_fragment_table_4(long long *directory_table_end)
+ {
+ 	int res, i;
+-	int bytes = SQUASHFS_FRAGMENT_BYTES(sBlk.s.fragments);
+-	int  indexes = SQUASHFS_FRAGMENT_INDEXES(sBlk.s.fragments);
+-	long long fragment_table_index[indexes];
++	size_t bytes = SQUASHFS_FRAGMENT_BYTES(sBlk.s.fragments);
++	size_t indexes = SQUASHFS_FRAGMENT_INDEXES(sBlk.s.fragments);
++	long long *fragment_table_index;
+
+ 	TRACE("read_fragment_table: %d fragments, reading %d fragment indexes "
+ 		"from 0x%llx\n", sBlk.s.fragments, indexes,
+@@ -44,6 +44,11 @@ int read_fragment_table_4(long long *directory_table_end)
+ 		return TRUE;
+ 	}
+
++	fragment_table_index = malloc(indexes*sizeof(long long));
++	if(fragment_table_index == NULL)
++		EXIT_UNSQUASH("read_fragment_table: failed to allocate "
++			"fragment table index\n");
++
+ 	fragment_table = malloc(bytes);
+ 	if(fragment_table == NULL)
+ 		EXIT_UNSQUASH("read_fragment_table: failed to allocate "

finalize.patch

@@ -0,0 +1,12 @@
+--- a/squashfs-tools/mksquashfs.c
++++ b/squashfs-tools/mksquashfs.c
+@@ -2445,8 +2445,8 @@ void *frag_deflator(void *arg)
+ 			write_buffer->block = bytes;
+ 			bytes += compressed_size;
+ 			fragments_outstanding --;
+-			pthread_mutex_unlock(&fragment_mutex);
+ 			queue_put(to_writer, write_buffer);
++			pthread_mutex_unlock(&fragment_mutex);
+ 			TRACE("Writing fragment %lld, uncompressed size %d, "
+ 				"compressed size %d\n", file_buffer->block,
+ 				file_buffer->size, compressed_size);

local-cve-fix.patch

@@ -0,0 +1,11 @@
+--- squashfs-tools/unsquash-4.c.orig	2015-06-24 14:23:22.270710744 -0500
++++ squashfs-tools/unsquash-4.c	2015-06-24 14:24:13.671243487 -0500
+@@ -35,7 +35,7 @@
+ 	size_t indexes = SQUASHFS_FRAGMENT_INDEXES(sBlk.s.fragments);
+ 	long long *fragment_table_index;
+
+-	TRACE("read_fragment_table: %d fragments, reading %d fragment indexes "
++	TRACE("read_fragment_table: %u fragments, reading %zu fragment indexes "
+ 		"from 0x%llx\n", sBlk.s.fragments, indexes,
+ 		sBlk.s.fragment_table_start);
+