chunkfs: implementation of chunkfs by Valerie Henson <[email protected]>

From: http://valerieaurora.org/chunkfs/lca_release.diff

$ patch -p1 < patches/chunkfs_lca_release.diff
patching file fs/Kconfig
Hunk #1 FAILED at 272.
1 out of 1 hunk FAILED -- saving rejects to file fs/Kconfig.rej
patching file fs/Makefile
Hunk #1 FAILED at 73.
1 out of 1 hunk FAILED -- saving rejects to file fs/Makefile.rej
patching file fs/chunkfs/Makefile
patching file fs/chunkfs/chunkfs.h
patching file fs/chunkfs/mkfs.chunkfs.c
patching file fs/chunkfs/super.c
can't find file to patch at input line 981
Perhaps you used the wrong -p or --strip option?
The text leading up to this was:
--------------------------
|--- chunkfs-2.6-mm-linux.orig/include/linux/magic.h
|+++ chunkfs-2.6-mm-linux/include/linux/magic.h
--------------------------
File to patch: include/uapi/linux/magic.h
patching file include/uapi/linux/magic.h
Hunk #1 succeeded at 5 with fuzz 2.
patching file fs/chunkfs/README
patching file fs/chunkfs/chunkfs_chunk.h
patching file fs/chunkfs/chunkfs_dev.h
patching file fs/chunkfs/chunkfs_i.h
patching file fs/chunkfs/chunkfs_pool.h
patching file fs/chunkfs/dir.c
patching file fs/chunkfs/file.c
patching file fs/chunkfs/fsck.chunkfs.c
patching file fs/chunkfs/gaze.sh
patching file fs/chunkfs/inode.c
patching file fs/chunkfs/namei.c
patching file fs/chunkfs/start_uml.sh
patching file fs/chunkfs/symlink.c
patching file fs/chunkfs/test.sh
patching file fs/chunkfs/write_pattern.c
patching file fs/namespace.c
Hunk #1 FAILED at 340.
Hunk #2 FAILED at 351.
2 out of 2 hunks FAILED -- saving rejects to file fs/namespace.c.rej
patching file include/linux/mount.h
Hunk #1 FAILED at 67.
1 out of 1 hunk FAILED -- saving rejects to file
include/linux/mount.h.rej
patching file fs/chunkfs/cont.c
patching file fs/chunkfs/cross.sh
patching file fs/chunkfs/demo.sh

.gitignore

@@ -0,0 +1,97 @@
+# specific files
+mkfs.chunkfs
+write_pattern
+
+#
+# NOTE! Don't add files that are generated in specific
+# subdirectories here. Add them in the ".gitignore" file
+# in that subdirectory instead.
+#
+# NOTE! Please use 'git ls-files -i --exclude-standard'
+# command after changing this file, to see if there are
+# any tracked files which get ignored after the change.
+#
+# Normal rules
+#
+.*
+*.o
+*.o.*
+*.a
+*.s
+*.ko
+*.so
+*.so.dbg
+*.mod.c
+*.i
+*.lst
+*.symtypes
+*.order
+modules.builtin
+*.elf
+*.bin
+*.gz
+*.bz2
+*.lzma
+*.xz
+*.lzo
+*.patch
+*.gcno
+
+#
+# Top-level generic files
+#
+/tags
+/TAGS
+/linux
+/vmlinux
+/vmlinuz
+/System.map
+/Module.markers
+/Module.symvers
+
+#
+# Debian directory (make deb-pkg)
+#
+/debian/
+
+#
+# git files that we don't want to ignore even it they are dot-files
+#
+!.gitignore
+!.mailmap
+
+#
+# Generated include files
+#
+include/config
+include/generated
+arch/*/include/generated
+
+# stgit generated dirs
+patches-*
+
+# quilt's files
+patches
+series
+
+# cscope files
+cscope.*
+ncscope.*
+
+# gnu global files
+GPATH
+GRTAGS
+GSYMS
+GTAGS
+
+*.orig
+*~
+\#*#
+
+#
+# Leavings from module signing
+#
+extra_certificates
+signing_key.priv
+signing_key.x509
+x509.genkey

Makefile

@@ -0,0 +1,16 @@
+#
+# Makefile for chunkfs.
+#
+
+obj-$(CONFIG_CHUNK_FS) += chunkfs.o
+
+chunkfs-y	:= super.o inode.o dir.o file.o namei.o symlink.o cont.o
+
+#
+# Temporarily keep utilities in this dir too.
+#
+HOST_EXTRACFLAGS += -I$(src)/../../include -static
+
+hostprogs-$(CONFIG_CHUNK_FS)	:= mkfs.chunkfs mount_chunkfs write_pattern
+
+always          := $(hostprogs-y) $(hostprogs-m)

README

@@ -0,0 +1,70 @@
+Chunkfs README
+
+Val Henson <[email protected]>
+
+Summary
+-------
+
+Chunkfs is an experimental local file system designed to quickly
+recover from file system corruption.  Each file system is divided up
+into many small chunks, each of which can be checked and repaired with
+very few references to other chunks.  In most cases, only a small part
+of the file system must be checked and repaired before it can be
+brought back online, requiring minutes instead of hours of downtime to
+recover from a file system error.
+
+Status
+------
+
+Chunkfs development began in February 2007.  Growing a file into two
+chunks is supported but not much else.
+
+See the project web site for the current status:
+
+http://chunkfs.org
+
+License
+-------
+
+Chunkfs is licensed under the GNU General Public License version 2.
+Chunks is released as a patch against the Linux kernel, which contains
+a copy of the GPLv2.
+
+Funding
+-------
+
+Development of chunkfs was funded by:
+
+Intel
+EMC Centera
+VAH Consulting
+
+How it works
+------------
+
+Each file system is divided up into many small chunks.  Each chunk is
+marked as dirty or clean.  Things that make a chunk dirty are
+in-progress writes to metadata (creat(), chmod(), extending a file,
+etc.), I/O errors reported by the disk, and any data integrity errors
+observed by the file system code (bad checksums, wrong magic number,
+etc.).  At mount time, each dirty chunk is checked with fsck and
+repaired if necessary, with limited references to other chunks.  Clean
+chunks are not checked unless specifically requested.
+
+The metadata inside a chunk is structured so that nearly all
+references are within the chunk only.  No block pointers or hard links
+may cross chunk boundaries.  Only one kind of reference crosses the
+chunk boundary: inode continuations.  When an inode needs to grow out
+of a chunk (either its data has outgrown the free space or we want to
+link to an inode outside its chunk), we allocate a new inode in the
+appropriate chunk and link the two inodes together with forward and
+back pointers, creating a continuation.  Logically, the two inodes are
+parts of a single file or directory.  When checking a chunk containing
+a continuation inode, the forward and back pointers allow us to
+quickly find the relevant information in any other chunks, without
+reading all the metadata in the entire chunk.  Each chunk also keeps a
+bitmap of all inodes with continuations, as in some circumstances all
+continuations must be checked.
+
+For more information, see the documentation section on the project web
+site.

chunkfs.h

@@ -0,0 +1,204 @@
+/*
+ * Chunkfs header file
+ *
+ * (C) 2007-2008 Valerie Henson <[email protected]>
+ *
+ */
+
+#ifndef _LINUX_CHUNKFS_FS_H
+#define _LINUX_CHUNKFS_FS_H
+
+/* XXX Do above _H stuff for other header files */
+
+#include <linux/types.h>
+#include <linux/fs.h>
+#include <linux/limits.h>
+
+/*
+ * NOTE: Most on disk structures need:
+ *
+ * Magic number (32 bit is really big but easy)
+ * Checksum (32 bit for some kind of sanity)
+ *
+ * These go first (magic, then checksum) in all on-disk structures so
+ * that even if have the type of the structure wrong, we're checking
+ * the correct spot.
+ *
+ * XXX File system generation number should be included - perhaps high
+ * 16 of magic?
+ */
+
+/* XXX Should have magic, checksum, version, and generation in one struct */
+
+/*
+ * Some useful typedefs to possibly prevent endian mixups.  Use c_*
+ * for on disk, ci_* for in memory.
+ */
+
+typedef __le64 c_blk_t;
+typedef __u64 ci_blk_t;
+typedef __le64 c_byte_t;
+typedef __u64 ci_byte_t;
+typedef __le64 c_inode_num_t;
+typedef __u64 ci_inode_num_t;
+
+/*
+ * XXX Block size shouldn't have much meaning, and it will probably
+ * vary by chunk.  Figure out what Linux VFS thinks this means.
+ *
+ * I think this means that when you use sb_read(), this is the block
+ * size used.
+ */
+
+#define CHUNKFS_BLK_SIZE	4096
+#define CHUNKFS_BLK_BITS	12
+
+/*
+ * Rev me! Lots! Whenever on-disk structures change!  Mainly for
+ * development.
+ *
+ * Note that 0 is never acceptable.
+ */
+
+#define	CHUNKFS_VERSION	1
+
+/*
+ * XXX On-disk structures probably aren't correctly padded at any
+ * given moment in time.
+ */
+
+/*
+ * Locating a device has two parts.  First, we try a cached path name
+ * which is a hint only, since paths may change.  Then we check for
+ * the correct UUID; if it is wrong, we go search each device.
+ */
+
+#define CHUNKFS_DEV_PATH_LEN	1024
+
+struct chunkfs_dev_desc {
+	/*
+	 * The path of the device when last opened. It may have
+	 * changed, therefore it is only a hint.
+	 */
+	char d_hint[CHUNKFS_DEV_PATH_LEN];
+	/* Is this the device we're looking for? */
+	__le64 d_uuid;
+};
+
+/*
+ * Dummy struct to force us to check the "official" position of the
+ * checksum and magic number (at the beginning of the struct).
+ *
+ * XXX Actually use this struct in other structs.  Never access
+ * directly.
+ */
+
+struct chunkfs_chkmagic {
+	__le32 x_magic;
+	__le32 x_chksum;
+};
+
+/* XXX use e2fsprogs/dev uuid and crc32 lib functions */
+/* XXX using __cpu_to_* so userland can share */
+/* #ifdef KERNEL? How does Jeff do it? */
+
+static inline void write_chksum(void *buf, unsigned int size)
+{
+	struct chunkfs_chkmagic *x = (struct chunkfs_chkmagic *) buf;
+/*	x->x_chksum = __cpu_to_le32(crc32(buf, size)); */
+	x->x_chksum = __cpu_to_le32(0x32323232);
+}
+
+static inline int check_chksum(void *buf, unsigned int size)
+{
+	struct chunkfs_chkmagic *x = (struct chunkfs_chkmagic *) buf;
+/*	return !(x->x_chksum == __cpu_to_le32(crc32(buf, size))); */
+	return (__le32_to_cpu(x->x_chksum) != 0x32323232);
+}
+
+static inline int check_magic(void *buf, __u32 expected_magic) {
+	struct chunkfs_chkmagic *x = (struct chunkfs_chkmagic *) buf;
+	return (__le32_to_cpu(x->x_magic) != expected_magic);
+}
+/*
+ * Generic function to check a piece of metadata just read off disk.
+ * Checksum and magic number are -always- in the same location in all
+ * metadata.
+ */
+
+static inline int check_metadata(void *buf, unsigned int size, __u32 expected_magic)
+{
+	if (check_magic(buf, expected_magic))
+		return 1;
+	if (check_chksum(buf, size))
+		return 2;
+	return 0;
+}
+
+#ifdef __KERNEL__
+
+/* dir.c */
+extern struct file_operations chunkfs_dir_fops;
+
+/* inode.c */
+extern struct file_operations chunkfs_file_fops;
+extern struct inode_operations chunkfs_file_iops;
+int chunkfs_new_inode(struct super_block *, struct inode **);
+void chunkfs_start_inode(struct inode *inode, struct inode *client_inode,
+			 u64 chunk_id);
+void chunkfs_read_inode(struct inode *);
+int chunkfs_write_inode(struct inode *, int);
+void chunkfs_copy_up_inode(struct inode *, struct inode *);
+
+/* symlink.c */
+
+extern struct inode_operations chunkfs_symlink_iops;
+
+/* namei.c */
+
+extern struct inode_operations chunkfs_dir_iops;
+extern struct inode_operations chunkfs_special_iops;
+
+struct chunkfs_dlist_node *chunkfs_alloc_dlist_node(struct dentry *);
+void chunkfs_add_dentry(struct dentry *, struct dentry *, struct vfsmount *);
+int chunkfs_init_dentry(struct dentry *);
+void chunkfs_free_dentry(struct dentry *);
+void chunkfs_init_nd(struct inode *dir, struct dentry *dentry,
+		    struct dentry *client_dentry, u64 chunk_id);
+void chunkfs_copy_up_nd(struct nameidata *nd, struct nameidata *client_nd);
+void chunkfs_copy_down_nd(struct nameidata *nd, struct nameidata *client_nd);
+
+/* file.c */
+
+int chunkfs_setattr(struct dentry *dentry, struct iattr *attr);
+int chunkfs_permission(struct inode *, int, struct nameidata *);
+int chunkfs_open(struct inode *, struct file *);
+
+struct chunkfs_continuation;
+
+int chunkfs_open_cont_file(struct file *file, loff_t *ppos,
+			   struct file **client_file,
+			   struct chunkfs_continuation **ret_cont);
+void chunkfs_close_cont_file(struct file *file, struct file *client_file,
+			     struct chunkfs_continuation *cont);
+void chunkfs_copy_down_file(struct file *file, loff_t *ppos,
+			    struct file *client_file, u64 client_start);
+
+/* cont.c */
+
+int chunkfs_get_next_inode(struct inode *head_inode,
+			   struct inode *prev_inode, struct inode **ret_inode);
+int chunkfs_get_cont_at_offset(struct dentry *dentry, loff_t offset,
+			       struct chunkfs_continuation **ret_cont);
+int chunkfs_get_next_cont(struct dentry *head_dentry,
+			  struct chunkfs_continuation *prev_cont,
+			  struct chunkfs_continuation **next_cont);
+int chunkfs_create_continuation(struct file *file, loff_t *ppos,
+				struct file **client_file,
+				struct chunkfs_continuation **ret_cont);
+void chunkfs_put_continuation(struct chunkfs_continuation *cont);
+int chunkfs_init_cont_data(struct dentry *client_dentry);
+
+#endif	/* __KERNEL__ */
+
+#endif	/* _LINUX_CHUNKFS_FS_H */