Init commit.

Author: kmayerpatel

data/compressed.dat

@@ -0,0 +1,121 @@
+package ac;
+
+import java.io.IOException;
+
+import io.BitSource;
+import io.InsufficientBitsLeftException;
+
+public class ArithmeticDecoder<T> {
+	private SourceModel<T> _model;
+	private long _low;
+	private long _high;
+	private int _range_bit_width;
+	private long _range_mask;
+	private long _input_buffer;
+	private boolean _first_fill;
+	private int _bits_consumed;
+	private boolean _tracing;
+
+	public ArithmeticDecoder(SourceModel<T> model, int rangeBitWidth) {
+		this(model, rangeBitWidth, false);
+	}
+	
+	public ArithmeticDecoder(SourceModel<T> model, int rangeBitWidth, boolean tracing) {
+		assert rangeBitWidth < 63;
+		assert model != null;
+		
+		_range_bit_width = rangeBitWidth;
+		_model = model;
+		_low = 0;
+		_high = (0x1L << rangeBitWidth) - 1L;
+		_range_mask = ~(0xffffffffffffffffL << _range_bit_width);
+		_first_fill = true;
+		_input_buffer = 0;
+		_bits_consumed = 0;
+		_tracing = tracing;
+	}
+
+	public T decode(BitSource bit_source) throws InsufficientBitsLeftException, IOException {
+		if (_first_fill) {
+			for (int i=0; i<_range_bit_width; i++) {
+				_input_buffer = ((_input_buffer << 1) & _range_mask) | ((long) bit_source.next(1));
+			}
+			_first_fill = false;
+		}
+
+		long range_width =  _high - _low + 1;
+		int sym_idx = -1;
+		long sym_low = 0;
+		long sym_high = 0;
+
+		// Find the next symbol according to the symbol model.
+		
+		for (int i=0; i<_model.size(); i++) {
+			sym_low = _low + ((long) (range_width * _model.cdfLow(i)));
+			sym_high =_low + ((long) (range_width * _model.cdfHigh(i))) -1L;
+
+			if (_input_buffer >= sym_low && _input_buffer <= sym_high) {
+				// Found it
+				sym_idx = i;
+				break;
+			}
+		}
+		assert sym_idx != -1;
+
+		_low = sym_low;
+		_high = sym_high;
+
+		int high_bit = highBit(_input_buffer);
+		
+		while((highBit(_low) == high_bit) &&
+			  (highBit(_high) == high_bit)) {
+
+			_input_buffer = ((_input_buffer << 1) & _range_mask) | ((long) bit_source.next(1));
+			_low = (_low << 1) & _range_mask;
+			_high =((_high << 1) & _range_mask) | 0x1L;
+
+			high_bit = highBit(_input_buffer);
+			_bits_consumed++;
+		}
+
+		// Are we in the middle?	
+		long one_quarter_mark = (0x1L << _range_bit_width) / 4L;
+		long three_quarter_mark = one_quarter_mark * 3L;
+
+		
+		while (_low > three_quarter_mark && _high < one_quarter_mark) {
+			// Yes, so shift out the second highest bits until we are not.
+			
+			_input_buffer = ((_input_buffer & (_range_mask>>2)) << 1) | // Mask out top two bits and shift left
+					((long) bit_source.next(1)) |                       // Bring in next bit from source
+					(((long) high_bit) << (_range_bit_width-1));        // Restore top bit to old value
+
+			_low = ((_low & (_range_mask>>2)) << 1);                    // Mask out top two bits and shift left
+			                                                            // Low order bit already 0 from shift
+			                                                            // Top bit still 0 after mask
+			
+			_high = ((_high & (_range_mask>>2)) << 1) |                 // Mask out top two bits and shift left
+					0x1L |                                              // Set low order bit to 1
+					(0x1L << (_range_bit_width-1));                     // Restore top bit as 1.
+			
+			_bits_consumed++;
+		}
+
+		T symbol = _model.get(sym_idx);
+		
+		if (_tracing) {
+			System.out.println("Decoded: " + symbol.toString());
+			System.out.println("   High: " + String.format("%16x", _high));
+			System.out.println("    Low: " + String.format("%16x", _low));
+		}
+		return symbol;
+	}
+
+	public int getBitsConsumed() {
+		return _bits_consumed;
+	}
+	
+	private int highBit(long value) {
+		return (int) ((value >> (_range_bit_width-1)) & 0x1L);
+	}
+}

data/reuncompressed.txt

@@ -0,0 +1,117 @@
+package ac;
+
+import java.io.IOException;
+
+import io.BitSink;
+
+public class ArithmeticEncoder<T> {
+	private SourceModel<T> _model;
+	private long _low;
+	private long _high;
+	private int _pending;
+	private int _range_bit_width;
+	private long _range_mask;
+	private boolean _tracing;
+
+	public ArithmeticEncoder(SourceModel<T> model, int rangeBitWidth) {
+		this(model, rangeBitWidth, false);
+	}
+	
+	public ArithmeticEncoder(SourceModel<T> model, int rangeBitWidth, boolean tracing) {
+		assert rangeBitWidth < 63;
+		assert model != null;
+		
+		_range_bit_width = rangeBitWidth;
+		_model = model;
+		_low = 0;
+		_high = (0x1L << rangeBitWidth) - 1L;
+		_pending = 0;
+		_range_mask = ~(0xffffffffffffffffL << _range_bit_width);
+		_tracing = tracing;
+	}
+	
+	public int encode(T symbol, BitSink bitSink) throws IOException {
+		int num_bits_emitted = 0;
+		long range_width =  _high - _low + 1;
+
+		long old_low = _low;
+		
+		_low = old_low + ((long) (range_width * _model.cdfLow(symbol)));
+		_high = old_low + ((long) (range_width * _model.cdfHigh(symbol))) -1L; 
+		
+		assert _high > _low;
+		
+		// While top bit matches, emit bits
+		
+		while (highBit(_low) == highBit(_high)) {
+			int high_bit = highBit(_low);
+			bitSink.write(high_bit, 1);
+			num_bits_emitted++;
+			
+			// Write out pending bits if we have any.
+			
+			while (_pending > 0) {
+				bitSink.write(1-high_bit, 1);
+				num_bits_emitted++;
+				_pending--;
+			}
+
+			// Shift low and high
+			
+			_low = (_low << 1) & _range_mask;
+			_high =((_high << 1) | 0x1L) & _range_mask;
+		}
+		
+		// Are we in the middle?	
+		long one_quarter_mark = (0x1L << _range_bit_width) / 4L;
+		long three_quarter_mark = one_quarter_mark * 3L;
+		
+		while (_low > three_quarter_mark && _high < one_quarter_mark) {
+			// Yes, so shift out the second highest bit and accumulate pending bits
+
+			// We know that:
+			// _low must be in form  01xxxx...
+			// _high must be in form 10xxxx...
+			//
+			// To shift out the second bit, we'll mask out everything but the top two bits
+			// (i.e., the xxxx... part of above) and shift it over by 1 and then fix up the
+			// top bit and low order bit as follows:
+			// 
+			// _low needs its top bit to remain 0 and new low order bit to be 0
+			// _high needs its top bit to remain 1 and new low order bit to be 1
+
+			
+			_low = ((_low & (_range_mask>>2)) << 1);		  // Mask out all but top two bits and shift left one.
+															  // 0 comes in bottom as part of shift left.
+															  // Top bit still 0 after the mask as before.
+			
+
+			_high = ((_high & (_range_mask>>2)) << 1) |       // Mask out all but top two bits and shift left one
+					0x1L | 									  // Shift in 1 at the bottom.
+					(0x1L << (_range_bit_width-1));			  // Restore top bit to be a 1
+			
+			// Accumulate pending bits
+			_pending++;
+			
+		}
+
+		if (_tracing) {
+			System.out.println("Encoded: " + symbol.toString());
+			System.out.println("   High: " + String.format("%16x", _high));
+			System.out.println("    Low: " + String.format("%16x", _low));
+			System.out.println("Emitted: " + num_bits_emitted);
+		}
+		return num_bits_emitted;
+	}
+	
+	public void emitMiddle(BitSink bitSink) throws IOException {
+		bitSink.write("1");
+		for (int i=1; i<_range_bit_width; i++) {
+			bitSink.write("0");
+		}
+	}
+	
+	private int highBit(long value) {
+		return (int) ((value >> (_range_bit_width-1)) & 0x1L);
+	}	
+}

data/uncompressed.txt

@@ -0,0 +1,49 @@
+package ac;
+
+public interface SourceModel<T> {
+	
+	int size();
+
+	T get(int index);
+
+	default int lookup(T symbol) {
+		for (int i=0; i<size(); i++) {
+			if (get(i).equals(symbol)) {
+				return i;
+			}
+		}		
+		throw new RuntimeException("Symbol " + symbol.toString() + " not in model");
+	}
+
+	double cdfLow(int index);
+
+	default double cdfHigh(int index) {
+		if (index < 0 || index > size()-1) {
+			throw new IndexOutOfBoundsException();
+		}
+
+		if (index == size()-1) {
+			return 1.0;
+		} else {
+			return cdfLow(index+1);
+		}		
+	}
+
+	default double cdfLow(T symbol) {
+		return cdfLow(lookup(symbol));
+	}
+
+	default double cdfHigh(T symbol) {
+		return cdfHigh(lookup(symbol));
+	}
+	
+	default String cdfTable() {
+		String s = "";
+		
+		for (int i=0; i<size(); i++) {
+			s += "Symbol: " + get(i).toString() + " cdfLevel: " + cdfLow(i) + "\n";
+		}
+		
+		return s;
+	}
+}

src/ac/ArithmeticDecoder.java

@@ -0,0 +1,65 @@
+package app;
+
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.Scanner;
+
+import ac.ArithmeticDecoder;
+import io.InputStreamBitSource;
+import io.InsufficientBitsLeftException;
+
+public class ACDecodeTextFile {
+
+	public static void main(String[] args) throws InsufficientBitsLeftException, IOException {
+		String input_file_name = "data/compressed.dat";
+		String output_file_name = "data/reuncompressed.txt";
+
+		FileInputStream fis = new FileInputStream(input_file_name);
+
+		InputStreamBitSource bit_source = new InputStreamBitSource(fis);
+
+		// Read in symbol counts and set up model
+		
+		int[] symbol_counts = new int[256];
+		Integer[] symbols = new Integer[256];
+		
+		for (int i=0; i<256; i++) {
+			symbol_counts[i] = bit_source.next(32);
+			symbols[i] = i;
+		}
+
+		FreqCountIntegerSymbolModel model = new FreqCountIntegerSymbolModel(symbols, symbol_counts);
+		System.out.println(model.cdfTable());
+		
+		// Read in number of symbols encoded
+
+		int num_symbols = bit_source.next(32);
+
+		System.out.println("File has " + num_symbols + " symbols encoded");
+
+		// Read in range bit width and setup the decoder
+
+		int range_bit_width = bit_source.next(8);
+		ArithmeticDecoder<Integer> decoder = new ArithmeticDecoder<Integer>(model, range_bit_width);
+
+		// Decode and produce output.
+		
+		FileOutputStream fos = new FileOutputStream(output_file_name);
+
+		// Scanner s = new Scanner(System.in);
+		for (int i=0; i<num_symbols; i++) {
+			// s.next();
+			int sym = decoder.decode(bit_source);
+			fos.write(sym);
+			
+			if (i%10000 == 0) {
+				System.out.println("After " + i + " symbols, consumed " + decoder.getBitsConsumed());
+			}
+		}
+
+		fos.flush();
+		fos.close();
+		fis.close();
+	}
+}