Merge pull request #1775 from tgschultz/stdlib-serialization

Added serialization, bitstreams, traits for int sign, TagPayloadType, some fixes to std

std/io.zig

@@ -8,6 +8,8 @@ const debug = std.debug;
 const assert = debug.assert;
 const os = std.os;
 const mem = std.mem;
+const meta = std.meta;
+const trait = meta.trait;
 const Buffer = std.Buffer;
 const fmt = std.fmt;
 const File = std.os.File;
@@ -463,6 +465,153 @@ pub const SliceInStream = struct {
     }
 };
 
+/// Creates a stream which allows for reading bit fields from another stream
+pub fn BitInStream(endian: builtin.Endian, comptime Error: type) type {
+    return struct {
+        const Self = @This();
+
+        in_stream: *Stream,
+        bit_buffer: u7,
+        bit_count: u3,
+        stream: Stream,
+
+        pub const Stream = InStream(Error);
+        const u8_bit_count = comptime meta.bitCount(u8);
+        const u7_bit_count = comptime meta.bitCount(u7);
+        const u4_bit_count = comptime meta.bitCount(u4);
+
+        pub fn init(in_stream: *Stream) Self {
+            return Self{
+                .in_stream = in_stream,
+                .bit_buffer = 0,
+                .bit_count = 0,
+                .stream = Stream{ .readFn = read },
+            };
+        }
+
+        /// Reads `bits` bits from the stream and returns a specified unsigned int type
+        ///  containing them in the least significant end, returning an error if the
+        ///  specified number of bits could not be read.
+        pub fn readBitsNoEof(self: *Self, comptime U: type, bits: usize) !U {
+            var n: usize = undefined;
+            const result = try self.readBits(U, bits, &n);
+            if (n < bits) return error.EndOfStream;
+            return result;
+        }
+
+        /// Reads `bits` bits from the stream and returns a specified unsigned int type
+        ///  containing them in the least significant end. The number of bits successfully
+        ///  read is placed in `out_bits`, as reaching the end of the stream is not an error.
+        pub fn readBits(self: *Self, comptime U: type, bits: usize, out_bits: *usize) Error!U {
+            debug.assert(trait.isUnsignedInt(U));
+
+            //by extending the buffer to a minimum of u8 we can cover a number of edge cases
+            // related to shifting and casting.
+            const u_bit_count = comptime meta.bitCount(U);
+            const buf_bit_count = bc: {
+                debug.assert(u_bit_count >= bits);
+                break :bc if (u_bit_count <= u8_bit_count) u8_bit_count else u_bit_count;
+            };
+            const Buf = @IntType(false, buf_bit_count);
+            const BufShift = math.Log2Int(Buf);
+
+            out_bits.* = usize(0);
+            if (U == u0 or bits == 0) return 0;
+            var out_buffer = Buf(0);
+
+            if (self.bit_count > 0) {
+                const n = if (self.bit_count >= bits) @intCast(u3, bits) else self.bit_count;
+                const shift = u7_bit_count - n;
+                switch (endian) {
+                    builtin.Endian.Big => {
+                        out_buffer = Buf(self.bit_buffer >> shift);
+                        self.bit_buffer <<= n;
+                    },
+                    builtin.Endian.Little => {
+                        const value = (self.bit_buffer << shift) >> shift;
+                        out_buffer = Buf(value);
+                        self.bit_buffer >>= n;
+                    },
+                }
+                self.bit_count -= n;
+                out_bits.* = n;
+            }
+            //at this point we know bit_buffer is empty
+
+            //copy bytes until we have enough bits, then leave the rest in bit_buffer
+            while (out_bits.* < bits) {
+                const n = bits - out_bits.*;
+                const next_byte = self.in_stream.readByte() catch |err| {
+                    if (err == error.EndOfStream) {
+                        return @intCast(U, out_buffer);
+                    }
+                    //@BUG: See #1810. Not sure if the bug is that I have to do this for some
+                    // streams, or that I don't for streams with emtpy errorsets.
+                    return @errSetCast(Error, err);
+                };
+
+                switch (endian) {
+                    builtin.Endian.Big => {
+                        if (n >= u8_bit_count) {
+                            out_buffer <<= @intCast(u3, u8_bit_count - 1);
+                            out_buffer <<= 1;
+                            out_buffer |= Buf(next_byte);
+                            out_bits.* += u8_bit_count;
+                            continue;
+                        }
+
+                        const shift = @intCast(u3, u8_bit_count - n);
+                        out_buffer <<= @intCast(BufShift, n);
+                        out_buffer |= Buf(next_byte >> shift);
+                        out_bits.* += n;
+                        self.bit_buffer = @truncate(u7, next_byte << @intCast(u3, n - 1));
+                        self.bit_count = shift;
+                    },
+                    builtin.Endian.Little => {
+                        if (n >= u8_bit_count) {
+                            out_buffer |= Buf(next_byte) << @intCast(BufShift, out_bits.*);
+                            out_bits.* += u8_bit_count;
+                            continue;
+                        }
+
+                        const shift = @intCast(u3, u8_bit_count - n);
+                        const value = (next_byte << shift) >> shift;
+                        out_buffer |= Buf(value) << @intCast(BufShift, out_bits.*);
+                        out_bits.* += n;
+                        self.bit_buffer = @truncate(u7, next_byte >> @intCast(u3, n));
+                        self.bit_count = shift;
+                    },
+                }
+            }
+
+            return @intCast(U, out_buffer);
+        }
+
+        pub fn alignToByte(self: *Self) void {
+            self.bit_buffer = 0;
+            self.bit_count = 0;
+        }
+
+        pub fn read(self_stream: *Stream, buffer: []u8) Error!usize {
+            var self = @fieldParentPtr(Self, "stream", self_stream);
+
+            var out_bits: usize = undefined;
+            var out_bits_total = usize(0);
+            //@NOTE: I'm not sure this is a good idea, maybe alignToByte should be forced
+            if (self.bit_count > 0) {
+                for (buffer) |*b, i| {
+                    b.* = try self.readBits(u8, u8_bit_count, &out_bits);
+                    out_bits_total += out_bits;
+                }
+                const incomplete_byte = @boolToInt(out_bits_total % u8_bit_count > 0);
+                return (out_bits_total / u8_bit_count) + incomplete_byte;
+            }
+
+            return self.in_stream.read(buffer);
+        }
+    };
+}
+
 /// This is a simple OutStream that writes to a slice, and returns an error
 /// when it runs out of space.
 pub const SliceOutStream = struct {
@@ -656,6 +805,137 @@ pub const BufferOutStream = struct {
     }
 };
 
+/// Creates a stream which allows for writing bit fields to another stream
+pub fn BitOutStream(endian: builtin.Endian, comptime Error: type) type {
+    return struct {
+        const Self = @This();
+
+        out_stream: *Stream,
+        bit_buffer: u8,
+        bit_count: u4,
+        stream: Stream,
+
+        pub const Stream = OutStream(Error);
+        const u8_bit_count = comptime meta.bitCount(u8);
+        const u4_bit_count = comptime meta.bitCount(u4);
+
+        pub fn init(out_stream: *Stream) Self {
+            return Self{
+                .out_stream = out_stream,
+                .bit_buffer = 0,
+                .bit_count = 0,
+                .stream = Stream{ .writeFn = write },
+            };
+        }
+
+        /// Write the specified number of bits to the stream from the least significant bits of
+        ///  the specified unsigned int value. Bits will only be written to the stream when there
+        ///  are enough to fill a byte.
+        pub fn writeBits(self: *Self, value: var, bits: usize) Error!void {
+            if (bits == 0) return;
+
+            const U = @typeOf(value);
+            debug.assert(trait.isUnsignedInt(U));
+
+            //by extending the buffer to a minimum of u8 we can cover a number of edge cases
+            // related to shifting and casting.
+            const u_bit_count = comptime meta.bitCount(U);
+            const buf_bit_count = bc: {
+                debug.assert(u_bit_count >= bits);
+                break :bc if (u_bit_count <= u8_bit_count) u8_bit_count else u_bit_count;
+            };
+            const Buf = @IntType(false, buf_bit_count);
+            const BufShift = math.Log2Int(Buf);
+
+            const buf_value = @intCast(Buf, value);
+
+            const high_byte_shift = @intCast(BufShift, buf_bit_count - u8_bit_count);
+            var in_buffer = switch (endian) {
+                builtin.Endian.Big => buf_value << @intCast(BufShift, buf_bit_count - bits),
+                builtin.Endian.Little => buf_value,
+            };
+            var in_bits = bits;
+
+            if (self.bit_count > 0) {
+                const bits_remaining = u8_bit_count - self.bit_count;
+                const n = @intCast(u3, if (bits_remaining > bits) bits else bits_remaining);
+                switch (endian) {
+                    builtin.Endian.Big => {
+                        const shift = @intCast(BufShift, high_byte_shift + self.bit_count);
+                        const v = @intCast(u8, in_buffer >> shift);
+                        self.bit_buffer |= v;
+                        in_buffer <<= n;
+                    },
+                    builtin.Endian.Little => {
+                        const v = @truncate(u8, in_buffer) << @intCast(u3, self.bit_count);
+                        self.bit_buffer |= v;
+                        in_buffer >>= n;
+                    },
+                }
+                self.bit_count += n;
+                in_bits -= n;
+
+                //if we didn't fill the buffer, it's because bits < bits_remaining;
+                if (self.bit_count != u8_bit_count) return;
+                try self.out_stream.writeByte(self.bit_buffer);
+                self.bit_buffer = 0;
+                self.bit_count = 0;
+            }
+            //at this point we know bit_buffer is empty
+
+            //copy bytes until we can't fill one anymore, then leave the rest in bit_buffer
+            while (in_bits >= u8_bit_count) {
+                switch (endian) {
+                    builtin.Endian.Big => {
+                        const v = @intCast(u8, in_buffer >> high_byte_shift);
+                        try self.out_stream.writeByte(v);
+                        in_buffer <<= @intCast(u3, u8_bit_count - 1);
+                        in_buffer <<= 1;
+                    },
+                    builtin.Endian.Little => {
+                        const v = @truncate(u8, in_buffer);
+                        try self.out_stream.writeByte(v);
+                        in_buffer >>= @intCast(u3, u8_bit_count - 1);
+                        in_buffer >>= 1;
+                    },
+                }
+                in_bits -= u8_bit_count;
+            }
+
+            if (in_bits > 0) {
+                self.bit_count = @intCast(u4, in_bits);
+                self.bit_buffer = switch (endian) {
+                    builtin.Endian.Big => @truncate(u8, in_buffer >> high_byte_shift),
+                    builtin.Endian.Little => @truncate(u8, in_buffer),
+                };
+            }
+        }
+
+        /// Flush any remaining bits to the stream.
+        pub fn flushBits(self: *Self) !void {
+            if (self.bit_count == 0) return;
+            try self.out_stream.writeByte(self.bit_buffer);
+            self.bit_buffer = 0;
+            self.bit_count = 0;
+        }
+
+        pub fn write(self_stream: *Stream, buffer: []const u8) Error!void {
+            var self = @fieldParentPtr(Self, "stream", self_stream);
+
+            //@NOTE: I'm not sure this is a good idea, maybe flushBits should be forced
+            if (self.bit_count > 0) {
+                for (buffer) |b, i|
+                    try self.writeBits(b, u8_bit_count);
+                return;
+            }
+
+            return self.out_stream.write(buffer);
+        }
+    };
+}
+
+
+
 pub const BufferedAtomicFile = struct {
     atomic_file: os.AtomicFile,
     file_stream: os.File.OutStream,
@@ -696,11 +976,6 @@ pub const BufferedAtomicFile = struct {
     }
 };
 
-test "import io tests" {
-    comptime {
-        _ = @import("io_test.zig");
-    }
-}
 
 pub fn readLine(buf: *std.Buffer) ![]u8 {
     var stdin = try getStdIn();
@@ -772,3 +1047,364 @@ test "io.readLineSliceFrom" {
     debug.assert(mem.eql(u8, "Line 1", try readLineSliceFrom(stream, buf[0..])));
     debug.assertError(readLineSliceFrom(stream, buf[0..]), error.OutOfMemory);
 }
+
+/// Creates a deserializer that deserializes types from any stream.
+///  If `is_packed` is true, the data stream is treated as bit-packed,
+///  otherwise data is expected to be packed to the smallest byte.
+///  Types may implement a custom deserialization routine with a
+///  function named `deserialize` in the form of:
+///    pub fn deserialize(self: *Self, deserializer: var) !void
+///  which will be called when the deserializer is used to deserialize
+///  that type. It will pass a pointer to the type instance to deserialize
+///  into and a pointer to the deserializer struct.
+pub fn Deserializer(endian: builtin.Endian, is_packed: bool, comptime Error: type) type {
+    return struct {
+        const Self = @This();
+
+        in_stream: if (is_packed) BitInStream(endian, Stream.Error) else *Stream,
+
+        pub const Stream = InStream(Error);
+
+        pub fn init(in_stream: *Stream) Self {
+            return Self{ .in_stream = switch (is_packed) {
+                true => BitInStream(endian, Stream.Error).init(in_stream),
+                else => in_stream,
+            } };
+        }
+        
+        pub fn alignToByte(self: *Self) void {
+            if(!is_packed) return;
+            self.in_stream.alignToByte();
+        }
+
+        //@BUG: inferred error issue. See: #1386 
+        fn deserializeInt(self: *Self, comptime T: type) (Stream.Error || error{EndOfStream})!T {
+            debug.assert(trait.is(builtin.TypeId.Int)(T) or trait.is(builtin.TypeId.Float)(T));
+
+            const u8_bit_count = comptime meta.bitCount(u8);
+            const t_bit_count = comptime meta.bitCount(T);
+
+            const U = @IntType(false, t_bit_count);
+            const Log2U = math.Log2Int(U);
+            const int_size = @sizeOf(U);
+
+            if (is_packed) {
+                const result = try self.in_stream.readBitsNoEof(U, t_bit_count);
+                return @bitCast(T, result);
+            }
+
+            var buffer: [int_size]u8 = undefined;
+            const read_size = try self.in_stream.read(buffer[0..]);
+            if (read_size < int_size) return error.EndOfStream;
+
+            if (int_size == 1) return @bitCast(T, buffer[0]);
+
+            var result = U(0);
+            for (buffer) |byte, i| {
+                switch (endian) {
+                    builtin.Endian.Big => {
+                        result = (result << @intCast(u4, u8_bit_count)) | byte;
+                    },
+                    builtin.Endian.Little => {
+                        result |= U(byte) << @intCast(Log2U, u8_bit_count * i);
+                    },
+                }
+            }
+
+            return @bitCast(T, result);
+        }
+
+        //@TODO: Replace this with @unionInit or whatever when it is added
+        // see: #1315
+        fn setTag(ptr: var, tag: var) void {
+            const T = @typeOf(ptr);
+            comptime debug.assert(trait.isPtrTo(builtin.TypeId.Union)(T));
+            const U = meta.Child(T);
+
+            const info = @typeInfo(U).Union;
+            if (info.tag_type) |TagType| {
+                debug.assert(TagType == @typeOf(tag));
+
+                var ptr_tag = ptr: {
+                    if (@alignOf(TagType) >= @alignOf(U)) break :ptr @ptrCast(*TagType, ptr);
+                    const offset = comptime max: {
+                        var max_field_size: comptime_int = 0;
+                        for (info.fields) |field_info| {
+                            const field_size = @sizeOf(field_info.field_type);
+                            max_field_size = math.max(max_field_size, field_size);
+                        }
+                        break :max math.max(max_field_size, @alignOf(U));
+                    };
+                    break :ptr @intToPtr(*TagType, @ptrToInt(ptr) + offset);
+                };
+                ptr_tag.* = tag;
+            }
+        }
+
+        /// Deserializes and returns data of the specified type from the stream
+        pub fn deserialize(self: *Self, comptime T: type) !T {
+            var value: T = undefined;
+            try self.deserializeInto(&value);
+            return value;
+        }
+
+        /// Deserializes data into the type pointed to by `ptr`
+        pub fn deserializeInto(self: *Self, ptr: var) !void {
+            const T = @typeOf(ptr);
+            debug.assert(trait.is(builtin.TypeId.Pointer)(T));
+
+            if (comptime trait.isSlice(T) or comptime trait.isPtrTo(builtin.TypeId.Array)(T)) {
+                for (ptr) |*v|
+                    try self.deserializeInto(v);
+                return;
+            }
+
+            comptime debug.assert(trait.isSingleItemPtr(T));
+
+            const C = comptime meta.Child(T);
+            const child_type_id = @typeId(C);
+
+            //custom deserializer: fn(self: *Self, deserializer: var) !void
+            if (comptime trait.hasFn("deserialize")(C)) return C.deserialize(ptr, self);
+
+            if (comptime trait.isPacked(C) and !is_packed) {
+                var packed_deserializer = Deserializer(endian, true, Error).init(self.in_stream);
+                return packed_deserializer.deserializeInto(ptr);
+            }
+
+            switch (child_type_id) {
+                builtin.TypeId.Void => return,
+                builtin.TypeId.Bool => ptr.* = (try self.deserializeInt(u1)) > 0,
+                builtin.TypeId.Float, builtin.TypeId.Int => ptr.* = try self.deserializeInt(C),
+                builtin.TypeId.Struct => {
+                    const info = @typeInfo(C).Struct;
+
+                    inline for (info.fields) |*field_info| {
+                        const name = field_info.name;
+                        const FieldType = field_info.field_type;
+
+                        if (FieldType == void or FieldType == u0) continue;
+
+                        //it doesn't make any sense to read pointers
+                        if (comptime trait.is(builtin.TypeId.Pointer)(FieldType)) {
+                            @compileError("Will not " ++ "read field " ++ name ++ " of struct " ++
+                                @typeName(C) ++ " because it " ++ "is of pointer-type " ++
+                                @typeName(FieldType) ++ ".");
+                        }
+
+                        try self.deserializeInto(&@field(ptr, name));
+                    }
+                },
+                builtin.TypeId.Union => {
+                    const info = @typeInfo(C).Union;
+                    if (info.tag_type) |TagType| {
+                        //we avoid duplicate iteration over the enum tags
+                        // by getting the int directly and casting it without
+                        // safety. If it is bad, it will be caught anyway.
+                        const TagInt = @TagType(TagType);
+                        const tag = try self.deserializeInt(TagInt);
+
+                        {
+                            @setRuntimeSafety(false);
+                            //See: #1315
+                            setTag(ptr, @intToEnum(TagType, tag));
+                        }
+
+                        inline for (info.fields) |field_info| {
+                            if (field_info.enum_field.?.value == tag) {
+                                const name = field_info.name;
+                                const FieldType = field_info.field_type;
+                                @field(ptr, name) = FieldType(undefined);
+                                try self.deserializeInto(&@field(ptr, name));
+                                return;
+                            }
+                        }
+                        //This is reachable if the enum data is bad
+                        return error.InvalidEnumTag;
+                    }
+                    @compileError("Cannot meaningfully deserialize " ++ @typeName(C) ++
+                        " because it is an untagged union  Use a custom deserialize().");
+                },
+                builtin.TypeId.Optional => {
+                    const OC = comptime meta.Child(C);
+                    const exists = (try self.deserializeInt(u1)) > 0;
+                    if (!exists) {
+                        ptr.* = null;
+                        return;
+                    }
+
+                    //The way non-pointer optionals are implemented ensures a pointer to them
+                    // will point to the value. The flag is stored at the end of that data.
+                    var val_ptr = @ptrCast(*OC, ptr);
+                    try self.deserializeInto(val_ptr);
+                    //This bit ensures the null flag isn't set. Any actual copying should be
+                    // optimized out... I hope.
+                    ptr.* = val_ptr.*;
+                },
+                builtin.TypeId.Enum => {
+                    var value = try self.deserializeInt(@TagType(C));
+                    ptr.* = try meta.intToEnum(C, value);
+                },
+                else => {
+                    @compileError("Cannot deserialize " ++ @tagName(child_type_id) ++ " types (unimplemented).");
+                },
+            }
+        }
+    };
+}
+
+/// Creates a serializer that serializes types to any stream.
+///  If `is_packed` is true, the data will be bit-packed into the stream.
+///  Note that the you must call `serializer.flush()` when you are done
+///  writing bit-packed data in order ensure any unwritten bits are committed.
+///  If `is_packed` is false, data is packed to the smallest byte. In the case
+///  of packed structs, the struct will written bit-packed and with the specified
+///  endianess, after which data will resume being written at the next byte boundary.
+///  Types may implement a custom serialization routine with a
+///  function named `serialize` in the form of:
+///    pub fn serialize(self: Self, serializer: var) !void
+///  which will be called when the serializer is used to serialize that type. It will
+///  pass a const pointer to the type instance to be serialized and a pointer
+///  to the serializer struct.
+pub fn Serializer(endian: builtin.Endian, is_packed: bool, comptime Error: type) type {
+    return struct {
+        const Self = @This();
+
+        out_stream: if (is_packed) BitOutStream(endian, Stream.Error) else *Stream,
+
+        pub const Stream = OutStream(Error);
+
+        pub fn init(out_stream: *Stream) Self {
+            return Self{ .out_stream = switch (is_packed) {
+                true => BitOutStream(endian, Stream.Error).init(out_stream),
+                else => out_stream,
+            } };
+        }
+
+        /// Flushes any unwritten bits to the stream
+        pub fn flush(self: *Self) Stream.Error!void {
+            if (is_packed) return self.out_stream.flushBits();
+        }
+
+        fn serializeInt(self: *Self, value: var) !void {
+            const T = @typeOf(value);
+            debug.assert(trait.is(builtin.TypeId.Int)(T) or trait.is(builtin.TypeId.Float)(T));
+
+            const t_bit_count = comptime meta.bitCount(T);
+            const u8_bit_count = comptime meta.bitCount(u8);
+
+            const U = @IntType(false, t_bit_count);
+            const Log2U = math.Log2Int(U);
+            const int_size = @sizeOf(U);
+
+            const u_value = @bitCast(U, value);
+
+            if (is_packed) return self.out_stream.writeBits(u_value, t_bit_count);
+
+            var buffer: [int_size]u8 = undefined;
+            if (int_size == 1) buffer[0] = u_value;
+
+            for (buffer) |*byte, i| {
+                const idx = switch (endian) {
+                    builtin.Endian.Big => int_size - i - 1,
+                    builtin.Endian.Little => i,
+                };
+                const shift = @intCast(Log2U, idx * u8_bit_count);
+                const v = u_value >> shift;
+                byte.* = if (t_bit_count < u8_bit_count) v else @truncate(u8, v);
+            }
+
+            try self.out_stream.write(buffer);
+        }
+
+        /// Serializes the passed value into the stream
+        pub fn serialize(self: *Self, value: var) !void {
+            const T = comptime @typeOf(value);
+
+            if (comptime trait.isIndexable(T)) {
+                for (value) |v|
+                    try self.serialize(v);
+                return;
+            }
+
+            //custom serializer: fn(self: Self, serializer: var) !void
+            if (comptime trait.hasFn("serialize")(T)) return T.serialize(value, self);
+
+            if (comptime trait.isPacked(T) and !is_packed) {
+                var packed_serializer = Serializer(endian, true, Error).init(self.out_stream);
+                try packed_serializer.serialize(value);
+                try packed_serializer.flush();
+                return;
+            }
+
+            switch (@typeId(T)) {
+                builtin.TypeId.Void => return,
+                builtin.TypeId.Bool => try self.serializeInt(u1(@boolToInt(value))),
+                builtin.TypeId.Float, builtin.TypeId.Int => try self.serializeInt(value),
+                builtin.TypeId.Struct => {
+                    const info = @typeInfo(T);
+
+                    inline for (info.Struct.fields) |*field_info| {
+                        const name = field_info.name;
+                        const FieldType = field_info.field_type;
+
+                        if (FieldType == void or FieldType == u0) continue;
+
+                        //It doesn't make sense to write pointers
+                        if (comptime trait.is(builtin.TypeId.Pointer)(FieldType)) {
+                            @compileError("Will not " ++ "serialize field " ++ name ++
+                                " of struct " ++ @typeName(T) ++ " because it " ++
+                                "is of pointer-type " ++ @typeName(FieldType) ++ ".");
+                        }
+                        try self.serialize(@field(value, name));
+                    }
+                },
+                builtin.TypeId.Union => {
+                    const info = @typeInfo(T).Union;
+                    if (info.tag_type) |TagType| {
+                        const active_tag = meta.activeTag(value);
+                        try self.serialize(active_tag);
+                        //This inline loop is necessary because active_tag is a runtime
+                        // value, but @field requires a comptime value. Our alternative
+                        // is to check each field for a match
+                        inline for (info.fields) |field_info| {
+                            if (field_info.enum_field.?.value == @enumToInt(active_tag)) {
+                                const name = field_info.name;
+                                const FieldType = field_info.field_type;
+                                try self.serialize(@field(value, name));
+                                return;
+                            }
+                        }
+                        unreachable;
+                    }
+                    @compileError("Cannot meaningfully serialize " ++ @typeName(T) ++
+                        " because it is an untagged union  Use a custom serialize().");
+                },
+                builtin.TypeId.Optional => {
+                    if (value == null) {
+                        try self.serializeInt(u1(@boolToInt(false)));
+                        return;
+                    }
+                    try self.serializeInt(u1(@boolToInt(true)));
+
+                    const OC = comptime meta.Child(T);
+
+                    //The way non-pointer optionals are implemented ensures a pointer to them
+                    // will point to the value. The flag is stored at the end of that data.
+                    var val_ptr = @ptrCast(*const OC, &value);
+                    try self.serialize(val_ptr.*);
+                },
+                builtin.TypeId.Enum => {
+                    try self.serializeInt(@enumToInt(value));
+                },
+                else => @compileError("Cannot serialize " ++ @tagName(@typeId(T)) ++ " types (unimplemented)."),
+            }
+        }
+    };
+}
+
+test "import io tests" {
+    comptime {
+        _ = @import("io_test.zig");
+    }
+}

std/io_test.zig

@@ -1,5 +1,7 @@
 const std = @import("index.zig");
 const io = std.io;
+const meta = std.meta;
+const trait = std.trait;
 const DefaultPrng = std.rand.DefaultPrng;
 const assert = std.debug.assert;
 const assertError = std.debug.assertError;
@@ -132,3 +134,438 @@ test "SliceOutStream" {
     assertError(ss.stream.write("Hello world!"), error.OutOfSpace);
     assert(mem.eql(u8, ss.getWritten(), "Hello worl"));
 }
+
+test "BitInStream" {
+    const mem_be = []u8{ 0b11001101, 0b00001011 };
+    const mem_le = []u8{ 0b00011101, 0b10010101 };
+
+    var mem_in_be = io.SliceInStream.init(mem_be[0..]);
+    const InError = io.SliceInStream.Error;
+    var bit_stream_be = io.BitInStream(builtin.Endian.Big, InError).init(&mem_in_be.stream);
+
+    var out_bits: usize = undefined;
+
+    assert(1 == try bit_stream_be.readBits(u2, 1, &out_bits));
+    assert(out_bits == 1);
+    assert(2 == try bit_stream_be.readBits(u5, 2, &out_bits));
+    assert(out_bits == 2);
+    assert(3 == try bit_stream_be.readBits(u128, 3, &out_bits));
+    assert(out_bits == 3);
+    assert(4 == try bit_stream_be.readBits(u8, 4, &out_bits));
+    assert(out_bits == 4);
+    assert(5 == try bit_stream_be.readBits(u9, 5, &out_bits));
+    assert(out_bits == 5);
+    assert(1 == try bit_stream_be.readBits(u1, 1, &out_bits));
+    assert(out_bits == 1);
+
+    mem_in_be.pos = 0;
+    bit_stream_be.bit_count = 0;
+    assert(0b110011010000101 == try bit_stream_be.readBits(u15, 15, &out_bits));
+    assert(out_bits == 15);
+
+    mem_in_be.pos = 0;
+    bit_stream_be.bit_count = 0;
+    assert(0b1100110100001011 == try bit_stream_be.readBits(u16, 16, &out_bits));
+    assert(out_bits == 16);
+
+    _ = try bit_stream_be.readBits(u0, 0, &out_bits);
+    
+    assert(0 == try bit_stream_be.readBits(u1, 1, &out_bits));
+    assert(out_bits == 0);
+    assertError(bit_stream_be.readBitsNoEof(u1, 1), error.EndOfStream);
+
+    var mem_in_le = io.SliceInStream.init(mem_le[0..]);
+    var bit_stream_le = io.BitInStream(builtin.Endian.Little, InError).init(&mem_in_le.stream);
+
+    assert(1 == try bit_stream_le.readBits(u2, 1, &out_bits));
+    assert(out_bits == 1);
+    assert(2 == try bit_stream_le.readBits(u5, 2, &out_bits));
+    assert(out_bits == 2);
+    assert(3 == try bit_stream_le.readBits(u128, 3, &out_bits));
+    assert(out_bits == 3);
+    assert(4 == try bit_stream_le.readBits(u8, 4, &out_bits));
+    assert(out_bits == 4);
+    assert(5 == try bit_stream_le.readBits(u9, 5, &out_bits));
+    assert(out_bits == 5);
+    assert(1 == try bit_stream_le.readBits(u1, 1, &out_bits));
+    assert(out_bits == 1);
+
+    mem_in_le.pos = 0;
+    bit_stream_le.bit_count = 0;
+    assert(0b001010100011101 == try bit_stream_le.readBits(u15, 15, &out_bits));
+    assert(out_bits == 15);
+
+    mem_in_le.pos = 0;
+    bit_stream_le.bit_count = 0;
+    assert(0b1001010100011101 == try bit_stream_le.readBits(u16, 16, &out_bits));
+    assert(out_bits == 16);
+
+    _ = try bit_stream_le.readBits(u0, 0, &out_bits);
+    
+    assert(0 == try bit_stream_le.readBits(u1, 1, &out_bits));
+    assert(out_bits == 0);
+    assertError(bit_stream_le.readBitsNoEof(u1, 1), error.EndOfStream);
+}
+
+test "BitOutStream" {
+    var mem_be = []u8{0} ** 2;
+    var mem_le = []u8{0} ** 2;
+
+    var mem_out_be = io.SliceOutStream.init(mem_be[0..]);
+    const OutError = io.SliceOutStream.Error;
+    var bit_stream_be = io.BitOutStream(builtin.Endian.Big, OutError).init(&mem_out_be.stream);
+
+    try bit_stream_be.writeBits(u2(1), 1);
+    try bit_stream_be.writeBits(u5(2), 2);
+    try bit_stream_be.writeBits(u128(3), 3);
+    try bit_stream_be.writeBits(u8(4), 4);
+    try bit_stream_be.writeBits(u9(5), 5);
+    try bit_stream_be.writeBits(u1(1), 1);
+
+    assert(mem_be[0] == 0b11001101 and mem_be[1] == 0b00001011);
+
+    mem_out_be.pos = 0;
+
+    try bit_stream_be.writeBits(u15(0b110011010000101), 15);
+    try bit_stream_be.flushBits();
+    assert(mem_be[0] == 0b11001101 and mem_be[1] == 0b00001010);
+
+    mem_out_be.pos = 0;
+    try bit_stream_be.writeBits(u32(0b110011010000101), 16);
+    assert(mem_be[0] == 0b01100110 and mem_be[1] == 0b10000101);
+
+    try bit_stream_be.writeBits(u0(0), 0);
+
+    var mem_out_le = io.SliceOutStream.init(mem_le[0..]);
+    var bit_stream_le = io.BitOutStream(builtin.Endian.Little, OutError).init(&mem_out_le.stream);
+
+    try bit_stream_le.writeBits(u2(1), 1);
+    try bit_stream_le.writeBits(u5(2), 2);
+    try bit_stream_le.writeBits(u128(3), 3);
+    try bit_stream_le.writeBits(u8(4), 4);
+    try bit_stream_le.writeBits(u9(5), 5);
+    try bit_stream_le.writeBits(u1(1), 1);
+
+    assert(mem_le[0] == 0b00011101 and mem_le[1] == 0b10010101);
+
+    mem_out_le.pos = 0;
+    try bit_stream_le.writeBits(u15(0b110011010000101), 15);
+    try bit_stream_le.flushBits();
+    assert(mem_le[0] == 0b10000101 and mem_le[1] == 0b01100110);
+
+    mem_out_le.pos = 0;
+    try bit_stream_le.writeBits(u32(0b1100110100001011), 16);
+    assert(mem_le[0] == 0b00001011 and mem_le[1] == 0b11001101);
+
+    try bit_stream_le.writeBits(u0(0), 0);
+}
+
+test "BitStreams with File Stream" {
+    const tmp_file_name = "temp_test_file.txt";
+    {
+        var file = try os.File.openWrite(tmp_file_name);
+        defer file.close();
+
+        var file_out = file.outStream();
+        var file_out_stream = &file_out.stream;
+        const OutError = os.File.WriteError;
+        var bit_stream = io.BitOutStream(builtin.endian, OutError).init(file_out_stream);
+        
+        try bit_stream.writeBits(u2(1), 1);
+        try bit_stream.writeBits(u5(2), 2);
+        try bit_stream.writeBits(u128(3), 3);
+        try bit_stream.writeBits(u8(4), 4);
+        try bit_stream.writeBits(u9(5), 5);
+        try bit_stream.writeBits(u1(1), 1);
+        try bit_stream.flushBits();
+    }
+    {
+        var file = try os.File.openRead(tmp_file_name);
+        defer file.close();
+
+        var file_in = file.inStream();
+        var file_in_stream = &file_in.stream;
+        const InError = os.File.ReadError;
+        var bit_stream = io.BitInStream(builtin.endian, InError).init(file_in_stream);
+        
+        var out_bits: usize = undefined;
+
+        assert(1 == try bit_stream.readBits(u2, 1, &out_bits));
+        assert(out_bits == 1);
+        assert(2 == try bit_stream.readBits(u5, 2, &out_bits));
+        assert(out_bits == 2);
+        assert(3 == try bit_stream.readBits(u128, 3, &out_bits));
+        assert(out_bits == 3);
+        assert(4 == try bit_stream.readBits(u8, 4, &out_bits));
+        assert(out_bits == 4);
+        assert(5 == try bit_stream.readBits(u9, 5, &out_bits));
+        assert(out_bits == 5);
+        assert(1 == try bit_stream.readBits(u1, 1, &out_bits));
+        assert(out_bits == 1);
+        
+        assertError(bit_stream.readBitsNoEof(u1, 1), error.EndOfStream);
+    }
+    try os.deleteFile(tmp_file_name);
+}
+
+fn testIntSerializerDeserializer(comptime endian: builtin.Endian, comptime is_packed: bool) !void {
+    //@NOTE: if this test is taking too long, reduce the maximum tested bitsize
+    const max_test_bitsize = 128;
+    
+    const total_bytes = comptime blk: {
+        var bytes = 0;
+        comptime var i = 0;
+        while (i <= max_test_bitsize) : (i += 1) bytes += (i / 8) + @boolToInt(i % 8 > 0);
+        break :blk bytes * 2;
+    };
+    
+    var data_mem: [total_bytes]u8 = undefined;
+    var out = io.SliceOutStream.init(data_mem[0..]);
+    const OutError = io.SliceOutStream.Error;
+    var out_stream = &out.stream;
+    var serializer = io.Serializer(endian, is_packed, OutError).init(out_stream);
+
+    var in = io.SliceInStream.init(data_mem[0..]);
+    const InError = io.SliceInStream.Error;
+    var in_stream = &in.stream;
+    var deserializer = io.Deserializer(endian, is_packed, InError).init(in_stream);
+
+    comptime var i = 0;
+    inline while (i <= max_test_bitsize) : (i += 1) {
+        const U = @IntType(false, i);
+        const S = @IntType(true, i);
+        try serializer.serializeInt(U(i));
+        if (i != 0) try serializer.serializeInt(S(-1)) else try serializer.serialize(S(0));
+    }
+    try serializer.flush();
+
+    i = 0;
+    inline while (i <= max_test_bitsize) : (i += 1) {
+        const U = @IntType(false, i);
+        const S = @IntType(true, i);
+        const x = try deserializer.deserializeInt(U);
+        const y = try deserializer.deserializeInt(S);
+        assert(x == U(i));
+        if (i != 0) assert(y == S(-1)) else assert(y == 0);
+    }
+
+    const u8_bit_count = comptime meta.bitCount(u8);
+    //0 + 1 + 2 + ... n = (n * (n + 1)) / 2
+    //and we have each for unsigned and signed, so * 2
+    const total_bits = (max_test_bitsize * (max_test_bitsize + 1));
+    const extra_packed_byte = @boolToInt(total_bits % u8_bit_count > 0);
+    const total_packed_bytes = (total_bits / u8_bit_count) + extra_packed_byte;
+
+    assert(in.pos == if (is_packed) total_packed_bytes else total_bytes);
+}
+
+test "Serializer/Deserializer Int" {
+    try testIntSerializerDeserializer(builtin.Endian.Big, false);
+    try testIntSerializerDeserializer(builtin.Endian.Little, false);
+    try testIntSerializerDeserializer(builtin.Endian.Big, true);
+    try testIntSerializerDeserializer(builtin.Endian.Little, true);
+}
+
+fn testIntSerializerDeserializerInfNaN(comptime endian: builtin.Endian, 
+    comptime is_packed: bool) !void
+{
+    const mem_size = (16*2 + 32*2 + 64*2 + 128*2) / comptime meta.bitCount(u8);
+    var data_mem: [mem_size]u8 = undefined;
+    
+    var out = io.SliceOutStream.init(data_mem[0..]);
+    const OutError = io.SliceOutStream.Error;
+    var out_stream = &out.stream;
+    var serializer = io.Serializer(endian, is_packed, OutError).init(out_stream);
+
+    var in = io.SliceInStream.init(data_mem[0..]);
+    const InError = io.SliceInStream.Error;
+    var in_stream = &in.stream;
+    var deserializer = io.Deserializer(endian, is_packed, InError).init(in_stream);
+
+    //@TODO: isInf/isNan not currently implemented for f128.
+    try serializer.serialize(std.math.nan(f16));
+    try serializer.serialize(std.math.inf(f16));
+    try serializer.serialize(std.math.nan(f32));
+    try serializer.serialize(std.math.inf(f32));
+    try serializer.serialize(std.math.nan(f64));
+    try serializer.serialize(std.math.inf(f64));
+    //try serializer.serialize(std.math.nan(f128));
+    //try serializer.serialize(std.math.inf(f128));
+    const nan_check_f16 = try deserializer.deserialize(f16);
+    const inf_check_f16 = try deserializer.deserialize(f16);
+    const nan_check_f32 = try deserializer.deserialize(f32);
+    const inf_check_f32 = try deserializer.deserialize(f32);
+    const nan_check_f64 = try deserializer.deserialize(f64);
+    const inf_check_f64 = try deserializer.deserialize(f64);
+    //const nan_check_f128 = try deserializer.deserialize(f128);
+    //const inf_check_f128 = try deserializer.deserialize(f128);
+    assert(std.math.isNan(nan_check_f16));
+    assert(std.math.isInf(inf_check_f16));
+    assert(std.math.isNan(nan_check_f32));
+    assert(std.math.isInf(inf_check_f32));
+    assert(std.math.isNan(nan_check_f64));
+    assert(std.math.isInf(inf_check_f64));
+    //assert(std.math.isNan(nan_check_f128));
+    //assert(std.math.isInf(inf_check_f128));
+}
+
+test "Serializer/Deserializer Int: Inf/NaN" {
+    try testIntSerializerDeserializerInfNaN(builtin.Endian.Big, false);
+    try testIntSerializerDeserializerInfNaN(builtin.Endian.Little, false);
+    try testIntSerializerDeserializerInfNaN(builtin.Endian.Big, true);
+    try testIntSerializerDeserializerInfNaN(builtin.Endian.Little, true);
+}
+
+fn testAlternateSerializer(self: var, serializer: var) !void {
+    try serializer.serialize(self.f_f16);
+}
+
+fn testSerializerDeserializer(comptime endian: builtin.Endian, comptime is_packed: bool) !void {
+    const ColorType = enum(u4) {
+        RGB8 = 1,
+        RA16 = 2,
+        R32 = 3,
+    };
+
+    const TagAlign = union(enum(u32)) {
+        A: u8,
+        B: u8,
+        C: u8,
+    };
+
+    const Color = union(ColorType) {
+        RGB8: struct {
+            r: u8,
+            g: u8,
+            b: u8,
+            a: u8,
+        },
+        RA16: struct {
+            r: u16,
+            a: u16,
+        },
+        R32: u32,
+    };
+
+    const PackedStruct = packed struct {
+        f_i3: i3,
+        f_u2: u2,
+    };
+
+    
+    
+    //to test custom serialization
+    const Custom = struct {
+        f_f16: f16,
+        f_unused_u32: u32,
+
+        pub fn deserialize(self: *@This(), deserializer: var) !void {
+            try deserializer.deserializeInto(&self.f_f16);
+            self.f_unused_u32 = 47;
+        }
+
+        pub const serialize = testAlternateSerializer;
+    };
+
+    const MyStruct = struct {
+        f_i3: i3,
+        f_u8: u8,
+        f_tag_align: TagAlign,
+        f_u24: u24,
+        f_i19: i19,
+        f_void: void,
+        f_f32: f32,
+        f_f128: f128,
+        f_packed_0: PackedStruct,
+        f_i7arr: [10]i7,
+        f_of64n: ?f64,
+        f_of64v: ?f64,
+        f_color_type: ColorType,
+        f_packed_1: PackedStruct,
+        f_custom: Custom,
+        f_color: Color,
+    };
+
+    const my_inst = MyStruct{
+        .f_i3 = -1,
+        .f_u8 = 8,
+        .f_tag_align = TagAlign{ .B = 148 },
+        .f_u24 = 24,
+        .f_i19 = 19,
+        .f_void = {},
+        .f_f32 = 32.32,
+        .f_f128 = 128.128,
+        .f_packed_0 = PackedStruct{ .f_i3 = -1, .f_u2 = 2 },
+        .f_i7arr = [10]i7{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 },
+        .f_of64n = null,
+        .f_of64v = 64.64,
+        .f_color_type = ColorType.R32,
+        .f_packed_1 = PackedStruct{ .f_i3 = 1, .f_u2 = 1 },
+        .f_custom = Custom{ .f_f16 = 38.63, .f_unused_u32 = 47 },
+        .f_color = Color{ .R32 = 123822 },
+    };
+
+    var data_mem: [@sizeOf(MyStruct)]u8 = undefined;
+    var out = io.SliceOutStream.init(data_mem[0..]);
+    const OutError = io.SliceOutStream.Error;
+    var out_stream = &out.stream;
+    var serializer = io.Serializer(endian, is_packed, OutError).init(out_stream);
+
+    var in = io.SliceInStream.init(data_mem[0..]);
+    const InError = io.SliceInStream.Error;
+    var in_stream = &in.stream;
+    var deserializer = io.Deserializer(endian, is_packed, InError).init(in_stream);
+
+    try serializer.serialize(my_inst);
+
+    const my_copy = try deserializer.deserialize(MyStruct);
+    assert(meta.eql(my_copy, my_inst));
+}
+
+test "Serializer/Deserializer generic" {
+    try testSerializerDeserializer(builtin.Endian.Big, false);
+    try testSerializerDeserializer(builtin.Endian.Little, false);
+    try testSerializerDeserializer(builtin.Endian.Big, true);
+    try testSerializerDeserializer(builtin.Endian.Little, true);
+}
+
+fn testBadData(comptime endian: builtin.Endian, comptime is_packed: bool) !void {
+    const E = enum(u14) {
+        One = 1,
+        Two = 2,
+    };
+
+    const A = struct {
+        e: E,
+    };
+
+    const C = union(E) {
+        One: u14,
+        Two: f16,
+    };
+
+    var data_mem: [4]u8 = undefined;
+    var out = io.SliceOutStream.init(data_mem[0..]);
+    const OutError = io.SliceOutStream.Error;
+    var out_stream = &out.stream;
+    var serializer = io.Serializer(endian, is_packed, OutError).init(out_stream);
+
+    var in = io.SliceInStream.init(data_mem[0..]);
+    const InError = io.SliceInStream.Error;
+    var in_stream = &in.stream;
+    var deserializer = io.Deserializer(endian, is_packed, InError).init(in_stream);
+
+    try serializer.serialize(u14(3));
+    assertError(deserializer.deserialize(A), error.InvalidEnumTag);
+    out.pos = 0;
+    try serializer.serialize(u14(3));
+    try serializer.serialize(u14(88));
+    assertError(deserializer.deserialize(C), error.InvalidEnumTag);
+}
+
+test "Deserializer bad data" {
+    try testBadData(builtin.Endian.Big, false);
+    try testBadData(builtin.Endian.Little, false);
+    try testBadData(builtin.Endian.Big, true);
+    try testBadData(builtin.Endian.Little, true);
+}

std/meta/index.zig

@@ -95,7 +95,7 @@ test "std.meta.stringToEnum" {
     debug.assert(null == stringToEnum(E1, "C"));
 }
 
-pub fn bitCount(comptime T: type) u32 {
+pub fn bitCount(comptime T: type) comptime_int {
     return switch (@typeInfo(T)) {
         TypeId.Int => |info| info.bits,
         TypeId.Float => |info| info.bits,
@@ -108,7 +108,7 @@ test "std.meta.bitCount" {
     debug.assert(bitCount(f32) == 32);
 }
 
-pub fn alignment(comptime T: type) u29 {
+pub fn alignment(comptime T: type) comptime_int {
     //@alignOf works on non-pointer types
     const P = if (comptime trait.is(TypeId.Pointer)(T)) T else *T;
     return @typeInfo(P).Pointer.alignment;
@@ -386,6 +386,33 @@ test "std.meta.activeTag" {
     debug.assert(activeTag(u) == UE.Float);
 }
 
+///Given a tagged union type, and an enum, return the type of the union
+/// field corresponding to the enum tag.
+pub fn TagPayloadType(comptime U: type, tag: var) type {
+    const Tag = @typeOf(tag);
+    debug.assert(trait.is(builtin.TypeId.Union)(U));
+    debug.assert(trait.is(builtin.TypeId.Enum)(Tag));
+
+    const info = @typeInfo(U).Union;
+
+    inline for (info.fields) |field_info| {
+        if (field_info.enum_field.?.value == @enumToInt(tag)) return field_info.field_type;
+    }
+    unreachable;
+}
+
+test "std.meta.TagPayloadType" {
+    const Event = union(enum) {
+        Moved: struct {
+            from: i32,
+            to: i32,
+        },
+    };
+    const MovedEvent = TagPayloadType(Event, Event.Moved);
+    var e: Event = undefined;
+    debug.assert(MovedEvent == @typeOf(e.Moved));
+}
+
 ///Compares two of any type for equality. Containers are compared on a field-by-field basis,
 /// where possible. Pointers are not followed.
 pub fn eql(a: var, b: @typeOf(a)) bool {
@@ -439,6 +466,11 @@ pub fn eql(a: var, b: @typeOf(a)) bool {
                 builtin.TypeInfo.Pointer.Size.Slice => return a.ptr == b.ptr and a.len == b.len,
             }
         },
+        builtin.TypeId.Optional => {
+            if(a == null and b == null) return true;
+            if(a == null or b == null) return false;
+            return eql(a.?, b.?);
+        },
         else => return a == b,
     }
 }
@@ -452,7 +484,7 @@ test "std.meta.eql" {
 
     const U = union(enum) {
         s: S,
-        f: f32,
+        f: ?f32,
     };
 
     const s_1 = S{

std/meta/trait.zig

@@ -231,6 +231,37 @@ test "std.meta.trait.isPacked" {
     debug.assert(!isPacked(u8));
 }
 
+///
+pub fn isUnsignedInt(comptime T: type) bool {
+    return switch (@typeId(T)) {
+        builtin.TypeId.Int => !@typeInfo(T).Int.is_signed,
+        else => false,
+    };
+}
+
+test "isUnsignedInt" {
+    debug.assert(isUnsignedInt(u32) == true);
+    debug.assert(isUnsignedInt(comptime_int) == false);
+    debug.assert(isUnsignedInt(i64) == false);
+    debug.assert(isUnsignedInt(f64) == false);
+}
+
+///
+pub fn isSignedInt(comptime T: type) bool {
+    return switch (@typeId(T)) {
+        builtin.TypeId.ComptimeInt => true,
+        builtin.TypeId.Int => @typeInfo(T).Int.is_signed,
+        else => false,
+    };
+}
+
+test "isSignedInt" {
+    debug.assert(isSignedInt(u32) == false);
+    debug.assert(isSignedInt(comptime_int) == true);
+    debug.assert(isSignedInt(i64) == true);
+    debug.assert(isSignedInt(f64) == false);
+}
+
 ///
 pub fn isSingleItemPtr(comptime T: type) bool {
     if (comptime is(builtin.TypeId.Pointer)(T)) {