diff --git a/.gitignore b/.gitignore index 9a3b07e34612ffec682b53f07705047fb4861359..c1011c388308f7326c1388dd4bf2af720b3a0de6 100644 --- a/.gitignore +++ b/.gitignore @@ -47,6 +47,7 @@ snapshot.sh tags tests/go_gen tests/monsterdata_java_wire.mon +tests/monsterdata_java_wire_sp.mon tests/monsterdata_go_wire.mon tests/monsterdata_javascript_wire.mon tests/unicode_test.mon diff --git a/java/com/google/flatbuffers/ByteBufferUtil.java b/java/com/google/flatbuffers/ByteBufferUtil.java new file mode 100644 index 0000000000000000000000000000000000000000..624dc4e2f7c01207a875b2411bea16df55f1ad95 --- /dev/null +++ b/java/com/google/flatbuffers/ByteBufferUtil.java @@ -0,0 +1,58 @@ +/* + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.flatbuffers; + +import static com.google.flatbuffers.Constants.*; + +import java.nio.ByteBuffer; + +/// @file +/// @addtogroup flatbuffers_java_api +/// @{ + +/** + * Class that collects utility functions around `ByteBuffer`. + */ +public class ByteBufferUtil { + + /** + * Extract the size prefix from a `ByteBuffer`. + * + * @param bb a size-prefixed buffer + * @return the size prefix + */ + public static int getSizePrefix(ByteBuffer bb) { + return bb.getInt(bb.position()); + } + + /** + * Create a duplicate of a size-prefixed `ByteBuffer` that has its position + * advanced just past the size prefix. + * + * @param bb a size-prefixed buffer + * @return a new buffer on the same underlying data that has skipped the + * size prefix + */ + public static ByteBuffer removeSizePrefix(ByteBuffer bb) { + ByteBuffer s = bb.duplicate(); + s.position(s.position() + SIZE_PREFIX_LENGTH); + return s; + } + +} + +/// @} diff --git a/java/com/google/flatbuffers/Constants.java b/java/com/google/flatbuffers/Constants.java index f59063145e71ca31ff8b0b4855cb468c663fceac..751f4a6576c46f58259d01d1305690c5fb7cae71 100644 --- a/java/com/google/flatbuffers/Constants.java +++ b/java/com/google/flatbuffers/Constants.java @@ -37,6 +37,8 @@ public class Constants { static final int SIZEOF_DOUBLE = 8; /** The number of bytes in a file identifier. */ static final int FILE_IDENTIFIER_LENGTH = 4; + /** The number of bytes in a size prefix. */ + public static final int SIZE_PREFIX_LENGTH = 4; } /// @endcond diff --git a/java/com/google/flatbuffers/FlatBufferBuilder.java b/java/com/google/flatbuffers/FlatBufferBuilder.java index f98fbedcec462bf621d16128b8067f5720cdb7e7..44f107aead8d0885e3059f6eb1932a84fb2049a2 100644 --- a/java/com/google/flatbuffers/FlatBufferBuilder.java +++ b/java/com/google/flatbuffers/FlatBufferBuilder.java @@ -816,30 +816,75 @@ public class FlatBufferBuilder { * Finalize a buffer, pointing to the given `root_table`. * * @param root_table An offset to be added to the buffer. + * @param size_prefix Whether to prefix the size to the buffer. */ - public void finish(int root_table) { - prep(minalign, SIZEOF_INT); + protected void finish(int root_table, boolean size_prefix) { + prep(minalign, SIZEOF_INT + (size_prefix ? SIZEOF_INT : 0)); addOffset(root_table); + if (size_prefix) { + addInt(bb.capacity() - space); + } bb.position(space); finished = true; } + /** + * Finalize a buffer, pointing to the given `root_table`. + * + * @param root_table An offset to be added to the buffer. + */ + public void finish(int root_table) { + finish(root_table, false); + } + + /** + * Finalize a buffer, pointing to the given `root_table`, with the size prefixed. + * + * @param root_table An offset to be added to the buffer. + */ + public void finishSizePrefixed(int root_table) { + finish(root_table, true); + } + /** * Finalize a buffer, pointing to the given `root_table`. * * @param root_table An offset to be added to the buffer. * @param file_identifier A FlatBuffer file identifier to be added to the buffer before * `root_table`. + * @param size_prefix Whether to prefix the size to the buffer. */ - public void finish(int root_table, String file_identifier) { - prep(minalign, SIZEOF_INT + FILE_IDENTIFIER_LENGTH); + protected void finish(int root_table, String file_identifier, boolean size_prefix) { + prep(minalign, SIZEOF_INT + FILE_IDENTIFIER_LENGTH + (size_prefix ? SIZEOF_INT : 0)); if (file_identifier.length() != FILE_IDENTIFIER_LENGTH) throw new AssertionError("FlatBuffers: file identifier must be length " + FILE_IDENTIFIER_LENGTH); for (int i = FILE_IDENTIFIER_LENGTH - 1; i >= 0; i--) { addByte((byte)file_identifier.charAt(i)); } - finish(root_table); + finish(root_table, size_prefix); + } + + /** + * Finalize a buffer, pointing to the given `root_table`. + * + * @param root_table An offset to be added to the buffer. + * @param file_identifier A FlatBuffer file identifier to be added to the buffer before + * `root_table`. + */ + public void finish(int root_table, String file_identifier) { + finish(root_table, file_identifier, false); + } + + /** + * Finalize a buffer, pointing to the given `root_table`, with the size prefixed. + * + * @param root_table An offset to be added to the buffer. + * @param file_identifier A FlatBuffer file identifier to be added to the buffer before + * `root_table`. + */ + public void finishSizePrefixed(int root_table, String file_identifier) { + finish(root_table, file_identifier, true); } /** diff --git a/net/FlatBuffers/ByteBuffer.cs b/net/FlatBuffers/ByteBuffer.cs index 37a2c7e6ed3800153632424310cc44c0ebdf33ff..878e740b5b99e7d19e0418bb492545a6692a3796 100644 --- a/net/FlatBuffers/ByteBuffer.cs +++ b/net/FlatBuffers/ByteBuffer.cs @@ -30,6 +30,8 @@ // using System; +using System.IO; +using System.Text; namespace FlatBuffers { @@ -38,12 +40,12 @@ namespace FlatBuffers /// </summary> public class ByteBuffer { - private readonly byte[] _buffer; + protected byte[] _buffer; private int _pos; // Must track start of the buffer. public int Length { get { return _buffer.Length; } } - public byte[] Data { get { return _buffer; } } + public ByteBuffer(int size) : this(new byte[size]) { } public ByteBuffer(byte[] buffer) : this(buffer, 0) { } @@ -63,11 +65,64 @@ namespace FlatBuffers _pos = 0; } + // Create a new ByteBuffer on the same underlying data. + // The new ByteBuffer's position will be same as this buffer's. + public ByteBuffer Duplicate() + { + return new ByteBuffer(_buffer, Position); + } + + // Increases the size of the ByteBuffer, and copies the old data towards + // the end of the new buffer. + public void GrowFront(int newSize) + { + if ((Length & 0xC0000000) != 0) + throw new Exception( + "ByteBuffer: cannot grow buffer beyond 2 gigabytes."); + + if (newSize < Length) + throw new Exception("ByteBuffer: cannot truncate buffer."); + + byte[] newBuffer = new byte[newSize]; + Buffer.BlockCopy(_buffer, 0, newBuffer, newSize - Length, + Length); + _buffer = newBuffer; + } + + public byte[] ToArray(int pos, int len) + { + byte[] arr = new byte[len]; + Buffer.BlockCopy(_buffer, pos, arr, 0, len); + return arr; + } + + public byte[] ToSizedArray() + { + return ToArray(Position, Length - Position); + } + + public byte[] ToFullArray() + { + return ToArray(0, Length); + } + + public ArraySegment<byte> ToArraySegment(int pos, int len) + { + return new ArraySegment<byte>(_buffer, pos, len); + } + + public MemoryStream ToMemoryStream(int pos, int len) + { + return new MemoryStream(_buffer, pos, len); + } + +#if !UNSAFE_BYTEBUFFER // Pre-allocated helper arrays for convertion. private float[] floathelper = new[] { 0.0f }; private int[] inthelper = new[] { 0 }; private double[] doublehelper = new[] { 0.0 }; private ulong[] ulonghelper = new[] { 0UL }; +#endif // !UNSAFE_BYTEBUFFER // Helper functions for the unsafe version. static public ushort ReverseBytes(ushort input) @@ -136,7 +191,6 @@ namespace FlatBuffers } #endif // !UNSAFE_BYTEBUFFER - private void AssertOffsetAndLength(int offset, int length) { #if !BYTEBUFFER_NO_BOUNDS_CHECK @@ -171,6 +225,13 @@ namespace FlatBuffers PutByte(offset, value); } + public void PutStringUTF8(int offset, string value) + { + AssertOffsetAndLength(offset, value.Length); + Encoding.UTF8.GetBytes(value, 0, value.Length, + _buffer, offset); + } + #if UNSAFE_BYTEBUFFER // Unsafe but more efficient versions of Put*. public void PutShort(int offset, short value) @@ -321,6 +382,11 @@ namespace FlatBuffers return _buffer[index]; } + public string GetStringUTF8(int startPos, int len) + { + return Encoding.UTF8.GetString(_buffer, startPos, len); + } + #if UNSAFE_BYTEBUFFER // Unsafe but more efficient versions of Get*. public short GetShort(int offset) diff --git a/net/FlatBuffers/ByteBuffer.exe b/net/FlatBuffers/ByteBuffer.exe new file mode 100755 index 0000000000000000000000000000000000000000..4361c00f2a116cc1336d27077e71c7f11c21e063 Binary files /dev/null and b/net/FlatBuffers/ByteBuffer.exe differ diff --git a/net/FlatBuffers/ByteBufferUtil.cs b/net/FlatBuffers/ByteBufferUtil.cs new file mode 100644 index 0000000000000000000000000000000000000000..66e8266843ba67495974e2a44d58bcfdfc0837df --- /dev/null +++ b/net/FlatBuffers/ByteBufferUtil.cs @@ -0,0 +1,39 @@ +/* + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; + +namespace FlatBuffers +{ + /// <summary> + /// Class that collects utility functions around `ByteBuffer`. + /// </summary> + public class ByteBufferUtil + { + // Extract the size prefix from a `ByteBuffer`. + public static int GetSizePrefix(ByteBuffer bb) { + return bb.GetInt(bb.Position); + } + + // Create a duplicate of a size-prefixed `ByteBuffer` that has its position + // advanced just past the size prefix. + public static ByteBuffer RemoveSizePrefix(ByteBuffer bb) { + ByteBuffer s = bb.Duplicate(); + s.Position += FlatBufferConstants.SizePrefixLength; + return s; + } + } +} diff --git a/net/FlatBuffers/FlatBufferBuilder.cs b/net/FlatBuffers/FlatBufferBuilder.cs index 90627fdfdd52cce9df47d1c4fff1e8f549dcc4f6..a2224498dc38249336f65931969abb338c454b4c 100644 --- a/net/FlatBuffers/FlatBufferBuilder.cs +++ b/net/FlatBuffers/FlatBufferBuilder.cs @@ -59,7 +59,7 @@ namespace FlatBuffers throw new ArgumentOutOfRangeException("initialSize", initialSize, "Must be greater than zero"); _space = initialSize; - _bb = new ByteBuffer(new byte[initialSize]); + _bb = new ByteBuffer(initialSize); } /// <summary> @@ -99,18 +99,7 @@ namespace FlatBuffers // the end of the new buffer (since we build the buffer backwards). void GrowBuffer() { - var oldBuf = _bb.Data; - var oldBufSize = oldBuf.Length; - if ((oldBufSize & 0xC0000000) != 0) - throw new Exception( - "FlatBuffers: cannot grow buffer beyond 2 gigabytes."); - - var newBufSize = oldBufSize << 1; - var newBuf = new byte[newBufSize]; - - Buffer.BlockCopy(oldBuf, 0, newBuf, newBufSize - oldBufSize, - oldBufSize); - _bb = new ByteBuffer(newBuf, newBufSize); + _bb.GrowFront(_bb.Length << 1); } // Prepare to write an element of `size` after `additional_bytes` @@ -475,7 +464,7 @@ namespace FlatBuffers AddByte(0); var utf8StringLen = Encoding.UTF8.GetByteCount(s); StartVector(1, utf8StringLen, 1); - Encoding.UTF8.GetBytes(s, 0, s.Length, _bb.Data, _space -= utf8StringLen); + _bb.PutStringUTF8(_space -= utf8StringLen, s); return new StringOffset(EndVector().Value); } @@ -586,13 +575,41 @@ namespace FlatBuffers /// <param name="rootTable"> /// An offset to be added to the buffer. /// </param> - public void Finish(int rootTable) + /// <param name="sizePrefix"> + /// Whether to prefix the size to the buffer. + /// </param> + protected void Finish(int rootTable, bool sizePrefix) { - Prep(_minAlign, sizeof(int)); + Prep(_minAlign, sizeof(int) + (sizePrefix ? sizeof(int) : 0)); AddOffset(rootTable); + if (sizePrefix) { + AddInt(_bb.Length - _space); + } _bb.Position = _space; } + /// <summary> + /// Finalize a buffer, pointing to the given `root_table`. + /// </summary> + /// <param name="rootTable"> + /// An offset to be added to the buffer. + /// </param> + public void Finish(int rootTable) + { + Finish(rootTable, false); + } + + /// <summary> + /// Finalize a buffer, pointing to the given `root_table`, with the size prefixed. + /// </summary> + /// <param name="rootTable"> + /// An offset to be added to the buffer. + /// </param> + public void FinishSizePrefixed(int rootTable) + { + Finish(rootTable, true); + } + /// <summary> /// Get the ByteBuffer representing the FlatBuffer. /// </summary> @@ -615,41 +632,69 @@ namespace FlatBuffers /// </returns> public byte[] SizedByteArray() { - var newArray = new byte[_bb.Data.Length - _bb.Position]; - Buffer.BlockCopy(_bb.Data, _bb.Position, newArray, 0, - _bb.Data.Length - _bb.Position); - return newArray; - } - - /// <summary> - /// Finalize a buffer, pointing to the given `rootTable`. - /// </summary> - /// <param name="rootTable"> - /// An offset to be added to the buffer. - /// </param> - /// <param name="fileIdentifier"> - /// A FlatBuffer file identifier to be added to the buffer before - /// `root_table`. - /// </param> - public void Finish(int rootTable, string fileIdentifier) - { - Prep(_minAlign, sizeof(int) + - FlatBufferConstants.FileIdentifierLength); - if (fileIdentifier.Length != - FlatBufferConstants.FileIdentifierLength) - throw new ArgumentException( - "FlatBuffers: file identifier must be length " + - FlatBufferConstants.FileIdentifierLength, - "fileIdentifier"); - for (int i = FlatBufferConstants.FileIdentifierLength - 1; i >= 0; - i--) - { - AddByte((byte)fileIdentifier[i]); - } - Finish(rootTable); + return _bb.ToSizedArray(); + } + + /// <summary> + /// Finalize a buffer, pointing to the given `rootTable`. + /// </summary> + /// <param name="rootTable"> + /// An offset to be added to the buffer. + /// </param> + /// <param name="fileIdentifier"> + /// A FlatBuffer file identifier to be added to the buffer before + /// `root_table`. + /// </param> + /// <param name="sizePrefix"> + /// Whether to prefix the size to the buffer. + /// </param> + protected void Finish(int rootTable, string fileIdentifier, bool sizePrefix) + { + Prep(_minAlign, sizeof(int) + (sizePrefix ? sizeof(int) : 0) + + FlatBufferConstants.FileIdentifierLength); + if (fileIdentifier.Length != + FlatBufferConstants.FileIdentifierLength) + throw new ArgumentException( + "FlatBuffers: file identifier must be length " + + FlatBufferConstants.FileIdentifierLength, + "fileIdentifier"); + for (int i = FlatBufferConstants.FileIdentifierLength - 1; i >= 0; + i--) + { + AddByte((byte)fileIdentifier[i]); + } + Finish(rootTable, sizePrefix); } + /// <summary> + /// Finalize a buffer, pointing to the given `rootTable`. + /// </summary> + /// <param name="rootTable"> + /// An offset to be added to the buffer. + /// </param> + /// <param name="fileIdentifier"> + /// A FlatBuffer file identifier to be added to the buffer before + /// `root_table`. + /// </param> + public void Finish(int rootTable, string fileIdentifier) + { + Finish(rootTable, fileIdentifier, false); + } + /// <summary> + /// Finalize a buffer, pointing to the given `rootTable`, with the size prefixed. + /// </summary> + /// <param name="rootTable"> + /// An offset to be added to the buffer. + /// </param> + /// <param name="fileIdentifier"> + /// A FlatBuffer file identifier to be added to the buffer before + /// `root_table`. + /// </param> + public void FinishSizePrefixed(int rootTable, string fileIdentifier) + { + Finish(rootTable, fileIdentifier, true); + } } } diff --git a/net/FlatBuffers/FlatBufferConstants.cs b/net/FlatBuffers/FlatBufferConstants.cs index ab3092c484c8f8f27e9fa7dcebdcd9c3eba6befd..e30f3f3944f11daa1c720dbcc5c5b8fa22b2289e 100644 --- a/net/FlatBuffers/FlatBufferConstants.cs +++ b/net/FlatBuffers/FlatBufferConstants.cs @@ -24,5 +24,6 @@ namespace FlatBuffers public static class FlatBufferConstants { public const int FileIdentifierLength = 4; + public const int SizePrefixLength = 4; } } diff --git a/net/FlatBuffers/Table.cs b/net/FlatBuffers/Table.cs index 55182b3846f0c468167c8c793585688e07d309bd..4a188ff887862d4a268d617e6d9e1d9bd093458c 100644 --- a/net/FlatBuffers/Table.cs +++ b/net/FlatBuffers/Table.cs @@ -60,7 +60,7 @@ namespace FlatBuffers offset += bb.GetInt(offset); var len = bb.GetInt(offset); var startPos = offset + sizeof(int); - return Encoding.UTF8.GetString(bb.Data, startPos , len); + return bb.GetStringUTF8(startPos, len); } // Get the length of a vector whose offset is stored at "offset" in this object. @@ -91,7 +91,7 @@ namespace FlatBuffers var pos = this.__vector(o); var len = this.__vector_len(o); - return new ArraySegment<byte>(this.bb.Data, pos, len); + return bb.ToArraySegment(pos, len); } // Initialize any Table-derived type to point to the union at the given offset. @@ -126,10 +126,11 @@ namespace FlatBuffers var startPos_1 = offset_1 + sizeof(int); var startPos_2 = offset_2 + sizeof(int); var len = Math.Min(len_1, len_2); - byte[] bbArray = bb.Data; for(int i = 0; i < len; i++) { - if (bbArray[i + startPos_1] != bbArray[i + startPos_2]) - return bbArray[i + startPos_1] - bbArray[i + startPos_2]; + byte b1 = bb.Get(i + startPos_1); + byte b2 = bb.Get(i + startPos_2); + if (b1 != b2) + return b1 - b2; } return len_1 - len_2; } @@ -142,10 +143,10 @@ namespace FlatBuffers var len_2 = key.Length; var startPos_1 = offset_1 + sizeof(int); var len = Math.Min(len_1, len_2); - byte[] bbArray = bb.Data; for (int i = 0; i < len; i++) { - if (bbArray[i + startPos_1] != key[i]) - return bbArray[i + startPos_1] - key[i]; + byte b = bb.Get(i + startPos_1); + if (b != key[i]) + return b - key[i]; } return len_1 - len_2; } diff --git a/python/flatbuffers/builder.py b/python/flatbuffers/builder.py index a21f3616164cfcadb9e111b2ee34481c197bf0ef..dea77ebfc321ca15a500c33f8701d0791f84c96a 100644 --- a/python/flatbuffers/builder.py +++ b/python/flatbuffers/builder.py @@ -483,14 +483,32 @@ class Builder(object): self.current_vtable[slotnum] = self.Offset() ## @endcond - def Finish(self, rootTable): + def __Finish(self, rootTable, sizePrefix): """Finish finalizes a buffer, pointing to the given `rootTable`.""" N.enforce_number(rootTable, N.UOffsetTFlags) - self.Prep(self.minalign, N.UOffsetTFlags.bytewidth) + prepSize = N.UOffsetTFlags.bytewidth + if sizePrefix: + prepSize += N.Int32Flags.bytewidth + self.Prep(self.minalign, prepSize) self.PrependUOffsetTRelative(rootTable) + if sizePrefix: + size = len(self.Bytes) - self.Head() + N.enforce_number(size, N.Int32Flags) + self.PrependInt32(size) self.finished = True return self.Head() + def Finish(self, rootTable): + """Finish finalizes a buffer, pointing to the given `rootTable`.""" + return self.__Finish(rootTable, False) + + def FinishSizePrefixed(self, rootTable): + """ + Finish finalizes a buffer, pointing to the given `rootTable`, + with the size prefixed. + """ + return self.__Finish(rootTable, True) + ## @cond FLATBUFFERS_INTERNAL def Prepend(self, flags, off): self.Prep(flags.bytewidth, 0) diff --git a/python/flatbuffers/util.py b/python/flatbuffers/util.py new file mode 100644 index 0000000000000000000000000000000000000000..abcb9c750ac4e28dd61d6d67ce5c81e13cd9e81b --- /dev/null +++ b/python/flatbuffers/util.py @@ -0,0 +1,28 @@ +# Copyright 2017 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from . import encode +from . import number_types +from . import packer + +def GetSizePrefix(buf, offset): + """Extract the size prefix from a buffer.""" + return encode.Get(packer.int32, buf, offset) + +def RemoveSizePrefix(buf, offset): + """ + Create a slice of a size-prefixed buffer that has + its position advanced just past the size prefix. + """ + return buf, offset + number_types.Int32Flags.bytewidth diff --git a/src/idl_gen_general.cpp b/src/idl_gen_general.cpp index 9c23c5dd36c6ca6100e63d0f825f80f3df512133..970f08ccdd8b1fcdd3a2613e525cb79755387648 100644 --- a/src/idl_gen_general.cpp +++ b/src/idl_gen_general.cpp @@ -1331,17 +1331,22 @@ class GeneralGenerator : public BaseGenerator { } code += " return " + GenOffsetConstruct(struct_def, "o") + ";\n }\n"; if (parser_.root_struct_def_ == &struct_def) { - code += " public static void "; - code += FunctionStart('F') + "inish" + struct_def.name; - code += - "Buffer(FlatBufferBuilder builder, " + GenOffsetType(struct_def); - code += " offset) {"; - code += " builder." + FunctionStart('F') + "inish(offset"; - if (lang_.language == IDLOptions::kCSharp) { code += ".Value"; } - - if (parser_.file_identifier_.length()) - code += ", \"" + parser_.file_identifier_ + "\""; - code += "); }\n"; + std::string size_prefix[] = { "", "SizePrefixed" }; + for (int i = 0; i < 2; ++i) { + code += " public static void "; + code += FunctionStart('F') + "inish" + size_prefix[i] + + struct_def.name; + code += "Buffer(FlatBufferBuilder builder, " + + GenOffsetType(struct_def); + code += " offset) {"; + code += " builder." + FunctionStart('F') + "inish" + size_prefix[i] + + "(offset"; + if (lang_.language == IDLOptions::kCSharp) { code += ".Value"; } + + if (parser_.file_identifier_.length()) + code += ", \"" + parser_.file_identifier_ + "\""; + code += "); }\n"; + } } } // Only generate key compare function for table, diff --git a/tests/FlatBuffers.Test/ByteBufferTests.cs b/tests/FlatBuffers.Test/ByteBufferTests.cs index 3324f12a3c23b3b1d3a189502fbf9b84c738d416..58bd71e59e7009cfdb17d569705c355b7bf3d7f2 100644 --- a/tests/FlatBuffers.Test/ByteBufferTests.cs +++ b/tests/FlatBuffers.Test/ByteBufferTests.cs @@ -44,8 +44,7 @@ namespace FlatBuffers.Test [FlatBuffersTestMethod] public void ByteBuffer_PutByteCannotPutAtOffsetPastLength() { - var buffer = new byte[1]; - var uut = new ByteBuffer(buffer); + var uut = new ByteBuffer(1); Assert.Throws<ArgumentOutOfRangeException>(() => uut.PutByte(1, 99)); } #endif @@ -66,8 +65,7 @@ namespace FlatBuffers.Test [FlatBuffersTestMethod] public void ByteBuffer_PutShortCannotPutAtOffsetPastLength() { - var buffer = new byte[2]; - var uut = new ByteBuffer(buffer); + var uut = new ByteBuffer(2); Assert.Throws<ArgumentOutOfRangeException>(() => uut.PutShort(2, 99)); } #endif @@ -76,16 +74,14 @@ namespace FlatBuffers.Test [FlatBuffersTestMethod] public void ByteBuffer_PutShortChecksLength() { - var buffer = new byte[1]; - var uut = new ByteBuffer(buffer); + var uut = new ByteBuffer(1); Assert.Throws<ArgumentOutOfRangeException>(() => uut.PutShort(0, 99)); } [FlatBuffersTestMethod] public void ByteBuffer_PutShortChecksLengthAndOffset() { - var buffer = new byte[2]; - var uut = new ByteBuffer(buffer); + var uut = new ByteBuffer(2); Assert.Throws<ArgumentOutOfRangeException>(() => uut.PutShort(1, 99)); } #endif @@ -108,24 +104,21 @@ namespace FlatBuffers.Test [FlatBuffersTestMethod] public void ByteBuffer_PutIntCannotPutAtOffsetPastLength() { - var buffer = new byte[4]; - var uut = new ByteBuffer(buffer); + var uut = new ByteBuffer(4); Assert.Throws<ArgumentOutOfRangeException>(() => uut.PutInt(2, 0x0A0B0C0D)); } [FlatBuffersTestMethod] public void ByteBuffer_PutIntChecksLength() { - var buffer = new byte[1]; - var uut = new ByteBuffer(buffer); + var uut = new ByteBuffer(1); Assert.Throws<ArgumentOutOfRangeException>(() => uut.PutInt(0, 0x0A0B0C0D)); } [FlatBuffersTestMethod] public void ByteBuffer_PutIntChecksLengthAndOffset() { - var buffer = new byte[4]; - var uut = new ByteBuffer(buffer); + var uut = new ByteBuffer(4); Assert.Throws<ArgumentOutOfRangeException>(() => uut.PutInt(2, 0x0A0B0C0D)); } #endif @@ -152,24 +145,21 @@ namespace FlatBuffers.Test [FlatBuffersTestMethod] public void ByteBuffer_PutLongCannotPutAtOffsetPastLength() { - var buffer = new byte[8]; - var uut = new ByteBuffer(buffer); + var uut = new ByteBuffer(8); Assert.Throws<ArgumentOutOfRangeException>(() => uut.PutLong(2, 0x010203040A0B0C0D)); } [FlatBuffersTestMethod] public void ByteBuffer_PutLongChecksLength() { - var buffer = new byte[1]; - var uut = new ByteBuffer(buffer); + var uut = new ByteBuffer(1); Assert.Throws<ArgumentOutOfRangeException>(() => uut.PutLong(0, 0x010203040A0B0C0D)); } [FlatBuffersTestMethod] public void ByteBuffer_PutLongChecksLengthAndOffset() { - var buffer = new byte[8]; - var uut = new ByteBuffer(buffer); + var uut = new ByteBuffer(8); Assert.Throws<ArgumentOutOfRangeException>(() => uut.PutLong(2, 0x010203040A0B0C0D)); } #endif @@ -187,8 +177,7 @@ namespace FlatBuffers.Test [FlatBuffersTestMethod] public void ByteBuffer_GetByteChecksOffset() { - var buffer = new byte[1]; - var uut = new ByteBuffer(buffer); + var uut = new ByteBuffer(1); Assert.Throws<ArgumentOutOfRangeException>(()=>uut.Get(1)); } #endif @@ -207,16 +196,14 @@ namespace FlatBuffers.Test [FlatBuffersTestMethod] public void ByteBuffer_GetShortChecksOffset() { - var buffer = new byte[2]; - var uut = new ByteBuffer(buffer); + var uut = new ByteBuffer(2); Assert.Throws<ArgumentOutOfRangeException>(() => uut.GetShort(2)); } [FlatBuffersTestMethod] public void ByteBuffer_GetShortChecksLength() { - var buffer = new byte[2]; - var uut = new ByteBuffer(buffer); + var uut = new ByteBuffer(2); Assert.Throws<ArgumentOutOfRangeException>(() => uut.GetShort(1)); } #endif @@ -237,16 +224,14 @@ namespace FlatBuffers.Test [FlatBuffersTestMethod] public void ByteBuffer_GetIntChecksOffset() { - var buffer = new byte[4]; - var uut = new ByteBuffer(buffer); + var uut = new ByteBuffer(4); Assert.Throws<ArgumentOutOfRangeException>(() => uut.GetInt(4)); } [FlatBuffersTestMethod] public void ByteBuffer_GetIntChecksLength() { - var buffer = new byte[2]; - var uut = new ByteBuffer(buffer); + var uut = new ByteBuffer(2); Assert.Throws<ArgumentOutOfRangeException>(() => uut.GetInt(0)); } #endif @@ -271,16 +256,14 @@ namespace FlatBuffers.Test [FlatBuffersTestMethod] public void ByteBuffer_GetLongChecksOffset() { - var buffer = new byte[8]; - var uut = new ByteBuffer(buffer); + var uut = new ByteBuffer(8); Assert.Throws<ArgumentOutOfRangeException>(() => uut.GetLong(8)); } [FlatBuffersTestMethod] public void ByteBuffer_GetLongChecksLength() { - var buffer = new byte[7]; - var uut = new ByteBuffer(buffer); + var uut = new ByteBuffer(7); Assert.Throws<ArgumentOutOfRangeException>(() => uut.GetLong(0)); } #endif @@ -317,5 +300,49 @@ namespace FlatBuffers.Test var rereverse = ByteBuffer.ReverseBytes(reverse); Assert.AreEqual(original, rereverse); } + + [FlatBuffersTestMethod] + public void ByteBuffer_ToFullArray_MatchesBuffer() + { + var buffer = new byte[4]; + buffer[0] = 0x0D; + buffer[1] = 0x0C; + buffer[2] = 0x0B; + buffer[3] = 0x0A; + var uut = new ByteBuffer(buffer); + Assert.ArrayEqual(buffer, uut.ToFullArray()); + } + + [FlatBuffersTestMethod] + public void ByteBuffer_ToSizedArray_MatchesBuffer() + { + var buffer = new byte[4]; + buffer[0] = 0x0D; + buffer[1] = 0x0C; + buffer[2] = 0x0B; + buffer[3] = 0x0A; + var uut = new ByteBuffer(buffer); + Assert.ArrayEqual(buffer, uut.ToFullArray()); + } + + [FlatBuffersTestMethod] + public void ByteBuffer_Duplicate_MatchesBuffer() + { + var buffer = new byte[4]; + buffer[0] = 0x0D; + buffer[1] = 0x0C; + buffer[2] = 0x0B; + buffer[3] = 0x0A; + var uut = new ByteBuffer(buffer); + Assert.AreEqual(0x0A0B0C0D, uut.GetInt(0)); + + // Advance by two bytes + uut.Position = 2; uut = uut.Duplicate(); + Assert.AreEqual(0x0A0B, uut.GetShort(2)); + + // Advance by one more byte + uut.Position = 1; uut = uut.Duplicate(); + Assert.AreEqual(0x0A, uut.Get(3)); + } } } diff --git a/tests/FlatBuffers.Test/FlatBuffers.Test.csproj b/tests/FlatBuffers.Test/FlatBuffers.Test.csproj index 38d62d6110f65d5d3c0a9e18e4e736f1c3924747..7c67274ae8648c590c6aabc4c8ec7e66564d52c2 100644 --- a/tests/FlatBuffers.Test/FlatBuffers.Test.csproj +++ b/tests/FlatBuffers.Test/FlatBuffers.Test.csproj @@ -41,6 +41,9 @@ <Compile Include="..\..\net\FlatBuffers\ByteBuffer.cs"> <Link>FlatBuffers\ByteBuffer.cs</Link> </Compile> + <Compile Include="..\..\net\FlatBuffers\ByteBufferUtil.cs"> + <Link>FlatBuffers\ByteBufferUtil.cs</Link> + </Compile> <Compile Include="..\..\net\FlatBuffers\IFlatbufferObject.cs"> <Link>FlatBuffers\IFlatbufferObject.cs</Link> </Compile> diff --git a/tests/FlatBuffers.Test/FlatBuffersExampleTests.cs b/tests/FlatBuffers.Test/FlatBuffersExampleTests.cs index ccd5b05decf8b6e80a09c2bf1c577f7c4428e4b9..82d4fdf2032e00a537ed5deb19bdd3520176ea43 100644 --- a/tests/FlatBuffers.Test/FlatBuffersExampleTests.cs +++ b/tests/FlatBuffers.Test/FlatBuffersExampleTests.cs @@ -32,6 +32,12 @@ namespace FlatBuffers.Test [FlatBuffersTestMethod] public void CanCreateNewFlatBufferFromScratch() + { + CanCreateNewFlatBufferFromScratch(true); + CanCreateNewFlatBufferFromScratch(false); + } + + private void CanCreateNewFlatBufferFromScratch(bool sizePrefix) { // Second, let's create a FlatBuffer from scratch in C#, and test it also. // We use an initial size of 1 to exercise the reallocation algorithm, @@ -95,22 +101,40 @@ namespace FlatBuffers.Test Monster.AddTestarrayoftables(fbb, sortMons); var mon = Monster.EndMonster(fbb); - Monster.FinishMonsterBuffer(fbb, mon); + if (sizePrefix) + { + Monster.FinishSizePrefixedMonsterBuffer(fbb, mon); + } + else + { + Monster.FinishMonsterBuffer(fbb, mon); + } // Dump to output directory so we can inspect later, if needed - using (var ms = new MemoryStream(fbb.DataBuffer.Data, fbb.DataBuffer.Position, fbb.Offset)) + using (var ms = fbb.DataBuffer.ToMemoryStream(fbb.DataBuffer.Position, fbb.Offset)) { var data = ms.ToArray(); - File.WriteAllBytes(@"Resources/monsterdata_cstest.mon",data); + string filename = @"Resources/monsterdata_cstest" + (sizePrefix ? "_sp" : "") + ".mon"; + File.WriteAllBytes(filename, data); + } + + // Remove the size prefix if necessary for further testing + ByteBuffer dataBuffer = fbb.DataBuffer; + if (sizePrefix) + { + Assert.AreEqual(ByteBufferUtil.GetSizePrefix(dataBuffer) + FlatBufferConstants.SizePrefixLength, + dataBuffer.Length - dataBuffer.Position); + dataBuffer = ByteBufferUtil.RemoveSizePrefix(dataBuffer); } // Now assert the buffer - TestBuffer(fbb.DataBuffer); + TestBuffer(dataBuffer); //Attempt to mutate Monster fields and check whether the buffer has been mutated properly // revert to original values after testing - Monster monster = Monster.GetRootAsMonster(fbb.DataBuffer); + Monster monster = Monster.GetRootAsMonster(dataBuffer); + // mana is optional and does not exist in the buffer so the mutation should fail // the mana field should retain its default value @@ -161,12 +185,12 @@ namespace FlatBuffers.Test pos.MutateX(1.0f); Assert.AreEqual(pos.X, 1.0f); - TestBuffer(fbb.DataBuffer); + TestBuffer(dataBuffer); } private void TestBuffer(ByteBuffer bb) { - var monster = Monster.GetRootAsMonster(bb); + Monster monster = Monster.GetRootAsMonster(bb); Assert.AreEqual(80, monster.Hp); Assert.AreEqual(150, monster.Mana); diff --git a/tests/FlatBuffers.Test/FlatBuffersFuzzTests.cs b/tests/FlatBuffers.Test/FlatBuffersFuzzTests.cs index d9332465616f38aab2e4ddb6d08fdfe123ef8589..2d411a2b22c3088282bd2c33499b7d50a5043f93 100644 --- a/tests/FlatBuffers.Test/FlatBuffersFuzzTests.cs +++ b/tests/FlatBuffers.Test/FlatBuffersFuzzTests.cs @@ -33,21 +33,21 @@ namespace FlatBuffers.Test public void TestNumbers() { var builder = new FlatBufferBuilder(1); - Assert.ArrayEqual(new byte[] { 0 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 0 }, builder.DataBuffer.ToFullArray()); builder.AddBool(true); - Assert.ArrayEqual(new byte[] { 1 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 1 }, builder.DataBuffer.ToFullArray()); builder.AddSbyte(-127); - Assert.ArrayEqual(new byte[] { 129, 1 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 129, 1 }, builder.DataBuffer.ToFullArray()); builder.AddByte(255); - Assert.ArrayEqual(new byte[] { 0, 255, 129, 1 }, builder.DataBuffer.Data); // First pad + Assert.ArrayEqual(new byte[] { 0, 255, 129, 1 }, builder.DataBuffer.ToFullArray()); // First pad builder.AddShort(-32222); - Assert.ArrayEqual(new byte[] { 0, 0, 0x22, 0x82, 0, 255, 129, 1 }, builder.DataBuffer.Data); // Second pad + Assert.ArrayEqual(new byte[] { 0, 0, 0x22, 0x82, 0, 255, 129, 1 }, builder.DataBuffer.ToFullArray()); // Second pad builder.AddUshort(0xFEEE); - Assert.ArrayEqual(new byte[] { 0xEE, 0xFE, 0x22, 0x82, 0, 255, 129, 1 }, builder.DataBuffer.Data); // no pad + Assert.ArrayEqual(new byte[] { 0xEE, 0xFE, 0x22, 0x82, 0, 255, 129, 1 }, builder.DataBuffer.ToFullArray()); // no pad builder.AddInt(-53687092); - Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 204, 204, 204, 252, 0xEE, 0xFE, 0x22, 0x82, 0, 255, 129, 1 }, builder.DataBuffer.Data); // third pad + Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 204, 204, 204, 252, 0xEE, 0xFE, 0x22, 0x82, 0, 255, 129, 1 }, builder.DataBuffer.ToFullArray()); // third pad builder.AddUint(0x98765432); - Assert.ArrayEqual(new byte[] { 0x32, 0x54, 0x76, 0x98, 204, 204, 204, 252, 0xEE, 0xFE, 0x22, 0x82, 0, 255, 129, 1 }, builder.DataBuffer.Data); // no pad + Assert.ArrayEqual(new byte[] { 0x32, 0x54, 0x76, 0x98, 204, 204, 204, 252, 0xEE, 0xFE, 0x22, 0x82, 0, 255, 129, 1 }, builder.DataBuffer.ToFullArray()); // no pad } [FlatBuffersTestMethod] @@ -55,11 +55,11 @@ namespace FlatBuffers.Test { var builder = new FlatBufferBuilder(1); builder.AddUlong(0x1122334455667788); - Assert.ArrayEqual(new byte[] { 0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11 }, builder.DataBuffer.ToFullArray()); builder = new FlatBufferBuilder(1); builder.AddLong(0x1122334455667788); - Assert.ArrayEqual(new byte[] { 0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11 }, builder.DataBuffer.ToFullArray()); } [FlatBuffersTestMethod] @@ -67,11 +67,11 @@ namespace FlatBuffers.Test { var builder = new FlatBufferBuilder(1); builder.StartVector(sizeof(byte), 1, 1); - Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 0, 0, 0, 0 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 0, 0, 0, 0 }, builder.DataBuffer.ToFullArray()); builder.AddByte(1); - Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 1, 0, 0, 0 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 1, 0, 0, 0 }, builder.DataBuffer.ToFullArray()); builder.EndVector(); - Assert.ArrayEqual(new byte[] { 1, 0, 0, 0, 1, 0, 0, 0 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 1, 0, 0, 0, 1, 0, 0, 0 }, builder.DataBuffer.ToFullArray()); } [FlatBuffersTestMethod] @@ -79,13 +79,13 @@ namespace FlatBuffers.Test { var builder = new FlatBufferBuilder(1); builder.StartVector(sizeof(byte), 2, 1); - Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 0, 0, 0, 0 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 0, 0, 0, 0 }, builder.DataBuffer.ToFullArray()); builder.AddByte(1); - Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 0, 1, 0, 0 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 0, 1, 0, 0 }, builder.DataBuffer.ToFullArray()); builder.AddByte(2); - Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 2, 1, 0, 0 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 2, 1, 0, 0 }, builder.DataBuffer.ToFullArray()); builder.EndVector(); - Assert.ArrayEqual(new byte[] { 2, 0, 0, 0, 2, 1, 0, 0 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 2, 0, 0, 0, 2, 1, 0, 0 }, builder.DataBuffer.ToFullArray()); } [FlatBuffersTestMethod] @@ -93,11 +93,11 @@ namespace FlatBuffers.Test { var builder = new FlatBufferBuilder(1); builder.StartVector(sizeof(ushort), 1, 1); - Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 0, 0, 0, 0 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 0, 0, 0, 0 }, builder.DataBuffer.ToFullArray()); builder.AddUshort(1); - Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 1, 0, 0, 0 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 1, 0, 0, 0 }, builder.DataBuffer.ToFullArray()); builder.EndVector(); - Assert.ArrayEqual(new byte[] { 1, 0, 0, 0, 1, 0, 0, 0 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 1, 0, 0, 0, 1, 0, 0, 0 }, builder.DataBuffer.ToFullArray()); } [FlatBuffersTestMethod] @@ -105,13 +105,13 @@ namespace FlatBuffers.Test { var builder = new FlatBufferBuilder(1); builder.StartVector(sizeof(ushort), 2, 1); - Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 0, 0, 0, 0 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 0, 0, 0, 0 }, builder.DataBuffer.ToFullArray()); builder.AddUshort(0xABCD); - Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 0, 0, 0xCD, 0xAB }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 0, 0, 0xCD, 0xAB }, builder.DataBuffer.ToFullArray()); builder.AddUshort(0xDCBA); - Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 0xBA, 0xDC, 0xCD, 0xAB }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 0, 0, 0, 0, 0xBA, 0xDC, 0xCD, 0xAB }, builder.DataBuffer.ToFullArray()); builder.EndVector(); - Assert.ArrayEqual(new byte[] { 2, 0, 0, 0, 0xBA, 0xDC, 0xCD, 0xAB }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 2, 0, 0, 0, 0xBA, 0xDC, 0xCD, 0xAB }, builder.DataBuffer.ToFullArray()); } [FlatBuffersTestMethod] @@ -119,7 +119,7 @@ namespace FlatBuffers.Test { var builder = new FlatBufferBuilder(1); builder.CreateString("foo"); - Assert.ArrayEqual(new byte[] { 3, 0, 0, 0, (byte)'f', (byte)'o', (byte)'o', 0 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 3, 0, 0, 0, (byte)'f', (byte)'o', (byte)'o', 0 }, builder.DataBuffer.ToFullArray()); builder.CreateString("moop"); Assert.ArrayEqual(new byte[] @@ -132,7 +132,7 @@ namespace FlatBuffers.Test 0, 0, 0, 0, // zero terminator with 3 byte pad 3, 0, 0, 0, (byte)'f', (byte)'o', (byte)'o', 0 - }, builder.DataBuffer.Data); + }, builder.DataBuffer.ToFullArray()); } [FlatBuffersTestMethod] @@ -144,7 +144,7 @@ namespace FlatBuffers.Test { 3, 0, 0, 0, 0x01, 0x02, 0x03, 0 - }, builder.DataBuffer.Data); // No padding + }, builder.DataBuffer.ToFullArray()); // No padding builder.CreateString("\x04\x05\x06\x07"); Assert.ArrayEqual(new byte[] { @@ -156,7 +156,7 @@ namespace FlatBuffers.Test 0, 0, 0, 0, // zero terminator with 3 byte pad 3, 0, 0, 0, 0x01, 0x02, 0x03, 0 - }, builder.DataBuffer.Data); // No padding + }, builder.DataBuffer.ToFullArray()); // No padding } [FlatBuffersTestMethod] @@ -164,14 +164,14 @@ namespace FlatBuffers.Test { var builder = new FlatBufferBuilder(1); builder.StartObject(0); - Assert.ArrayEqual(new byte[] { 0 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 0 }, builder.DataBuffer.ToFullArray()); builder.EndObject(); Assert.ArrayEqual(new byte[] { 4, 0, 4, 0, 4, 0, 0, 0 }, - builder.DataBuffer.Data); + builder.DataBuffer.ToFullArray()); } [FlatBuffersTestMethod] @@ -179,7 +179,7 @@ namespace FlatBuffers.Test { var builder = new FlatBufferBuilder(1); builder.StartObject(1); - Assert.ArrayEqual(new byte[] { 0 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 0 }, builder.DataBuffer.ToFullArray()); builder.AddBool(0, true, false); builder.EndObject(); Assert.ArrayEqual(new byte[] @@ -192,7 +192,7 @@ namespace FlatBuffers.Test 0, 0, 0, // padding 1, // value 0 }, - builder.DataBuffer.Data); + builder.DataBuffer.ToFullArray()); } [FlatBuffersTestMethod] @@ -200,7 +200,7 @@ namespace FlatBuffers.Test { var builder = new FlatBufferBuilder(1); builder.StartObject(1); - Assert.ArrayEqual(new byte[] { 0 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 0 }, builder.DataBuffer.ToFullArray()); builder.AddBool(0, false, false); builder.EndObject(); Assert.ArrayEqual(new byte[] @@ -211,7 +211,7 @@ namespace FlatBuffers.Test // entry 0 is not stored (trimmed end of vtable) 4, 0, 0, 0, // int32 offset for start of vtable }, - builder.DataBuffer.Data); + builder.DataBuffer.ToFullArray()); } [FlatBuffersTestMethod] @@ -219,7 +219,7 @@ namespace FlatBuffers.Test { var builder = new FlatBufferBuilder(1); builder.StartObject(1); - Assert.ArrayEqual(new byte[] { 0 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 0 }, builder.DataBuffer.ToFullArray()); builder.AddShort(0, 0x789A, 0); builder.EndObject(); Assert.ArrayEqual(new byte[] @@ -232,7 +232,7 @@ namespace FlatBuffers.Test 0, 0, // padding 0x9A, 0x78, //value 0 }, - builder.DataBuffer.Data); + builder.DataBuffer.ToFullArray()); } [FlatBuffersTestMethod] @@ -240,7 +240,7 @@ namespace FlatBuffers.Test { var builder = new FlatBufferBuilder(1); builder.StartObject(2); - Assert.ArrayEqual(new byte[] { 0 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 0 }, builder.DataBuffer.ToFullArray()); builder.AddShort(0, 0x3456, 0); builder.AddShort(1, 0x789A, 0); builder.EndObject(); @@ -254,7 +254,7 @@ namespace FlatBuffers.Test 0x9A, 0x78, // value 1 0x56, 0x34, // value 0 }, - builder.DataBuffer.Data); + builder.DataBuffer.ToFullArray()); } [FlatBuffersTestMethod] @@ -262,7 +262,7 @@ namespace FlatBuffers.Test { var builder = new FlatBufferBuilder(1); builder.StartObject(2); - Assert.ArrayEqual(new byte[] { 0 }, builder.DataBuffer.Data); + Assert.ArrayEqual(new byte[] { 0 }, builder.DataBuffer.ToFullArray()); builder.AddShort(0, 0x3456, 0); builder.AddBool(1, true, false); builder.EndObject(); @@ -276,7 +276,7 @@ namespace FlatBuffers.Test 0, 1, // padding + value 1 0x56, 0x34, // value 0 }, - builder.DataBuffer.Data); + builder.DataBuffer.ToFullArray()); } [FlatBuffersTestMethod] @@ -303,7 +303,7 @@ namespace FlatBuffers.Test 4, 0, 0, 0, 0, 0, 0, 0, }, - builder.DataBuffer.Data); + builder.DataBuffer.ToFullArray()); } [FlatBuffersTestMethod] @@ -330,7 +330,7 @@ namespace FlatBuffers.Test 0, 0, 55, 0, // value 0 0, 0, 0, 0, // length of vector (not in sctruc) }, - builder.DataBuffer.Data); + builder.DataBuffer.ToFullArray()); } @@ -361,7 +361,7 @@ namespace FlatBuffers.Test 0x78, 0x56, // vector value 0 0x34, 0x12, // vector value 1 }, - builder.DataBuffer.Data); + builder.DataBuffer.ToFullArray()); } [FlatBuffersTestMethod] @@ -391,7 +391,7 @@ namespace FlatBuffers.Test 0x00, 0x00, 0x34, 0x12, // struct value 1 0x00, 0x00, 0x00, 55, // struct value 0 }, - builder.DataBuffer.Data); + builder.DataBuffer.ToFullArray()); } [FlatBuffersTestMethod] @@ -425,7 +425,7 @@ namespace FlatBuffers.Test 44, // vector 0, 1 33, // vector 0, 0 }, - builder.DataBuffer.Data); + builder.DataBuffer.ToFullArray()); } [FlatBuffersTestMethod] @@ -438,7 +438,7 @@ namespace FlatBuffers.Test var off = builder.EndObject(); builder.Finish(off); - Assert.ArrayEqual(new byte[] + byte[] padded = new byte[] { 0, 0, 0, 0, 0, 0, 0, 0, @@ -452,8 +452,13 @@ namespace FlatBuffers.Test 66, 0, // value 1 0, 33, // value 0 - }, - builder.DataBuffer.Data); + }; + Assert.ArrayEqual(padded, builder.DataBuffer.ToFullArray()); + + // no padding in sized array + byte[] unpadded = new byte[padded.Length - 12]; + Buffer.BlockCopy(padded, 12, unpadded, 0, unpadded.Length); + Assert.ArrayEqual(unpadded, builder.DataBuffer.ToSizedArray()); } [FlatBuffersTestMethod] @@ -504,7 +509,7 @@ namespace FlatBuffers.Test 44, // value 1, 0 33, }, - builder.DataBuffer.Data); + builder.DataBuffer.ToFullArray()); } [FlatBuffersTestMethod] @@ -519,7 +524,7 @@ namespace FlatBuffers.Test var off = builder.EndObject(); builder.Finish(off); - Assert.ArrayEqual(new byte[] + byte[] padded = new byte[] { 0, 0, 0, 0, 0, 0, 0, 0, @@ -546,8 +551,61 @@ namespace FlatBuffers.Test 1, 1, 1, 1, // values 1, 1, 1, 1, - }, - builder.DataBuffer.Data); + }; + Assert.ArrayEqual(padded, builder.DataBuffer.ToFullArray()); + + // no padding in sized array + byte[] unpadded = new byte[padded.Length - 28]; + Buffer.BlockCopy(padded, 28, unpadded, 0, unpadded.Length); + Assert.ArrayEqual(unpadded, builder.DataBuffer.ToSizedArray()); + } + + [FlatBuffersTestMethod] + public void TestBunchOfBoolsSizePrefixed() + { + var builder = new FlatBufferBuilder(1); + builder.StartObject(8); + for (var i = 0; i < 8; i++) + { + builder.AddBool(i, true, false); + } + var off = builder.EndObject(); + builder.FinishSizePrefixed(off); + + byte[] padded = new byte[] + { + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, // padding to 64 bytes + + 36, 0, 0, 0, // size prefix + 24, 0, 0, 0, // root of table, pointing to vtable offset (obj0) + 20, 0, // vtable bytes + 12, 0, // object length + 11, 0, // start of value 0 + 10, 0, // start of value 1 + 9, 0, // start of value 2 + 8, 0, // start of value 3 + 7, 0, // start of value 4 + 6, 0, // start of value 5 + 5, 0, // start of value 6 + 4, 0, // start of value 7 + + 20, 0, 0, 0, // int32 offset for start of vtable + + 1, 1, 1, 1, // values + 1, 1, 1, 1, + + }; + Assert.ArrayEqual(padded, builder.DataBuffer.ToFullArray()); + + // no padding in sized array + byte[] unpadded = new byte[padded.Length - 24]; + Buffer.BlockCopy(padded, 24, unpadded, 0, unpadded.Length); + Assert.ArrayEqual(unpadded, builder.DataBuffer.ToSizedArray()); } [FlatBuffersTestMethod] @@ -569,7 +627,7 @@ namespace FlatBuffers.Test 0, 0, 128, 63, // value }, - builder.DataBuffer.Data); + builder.DataBuffer.ToFullArray()); } private void CheckObjects(int fieldCount, int objectCount) diff --git a/tests/FlatBuffers.Test/NetTest.sh b/tests/FlatBuffers.Test/NetTest.sh index ea16e47255431c013ba203d9c20918d39507eced..6201549c0c713f40fb4c667b0f9d6a9a5cd8d3e5 100644 --- a/tests/FlatBuffers.Test/NetTest.sh +++ b/tests/FlatBuffers.Test/NetTest.sh @@ -2,8 +2,22 @@ # Testing C# on Linux using Mono. -mcs -out:fbnettest.exe ../../net/FlatBuffers/*.cs ../MyGame/Example/*.cs FlatBuffersTestClassAttribute.cs FlatBuffersTestMethodAttribute.cs Assert.cs FlatBuffersExampleTests.cs Program.cs ByteBufferTests.cs FlatBufferBuilderTests.cs FlatBuffersFuzzTests.cs FuzzTestData.cs Lcg.cs TestTable.cs -./fbnettest.exe +mcs -debug -out:./fbnettest.exe \ + ../../net/FlatBuffers/*.cs ../MyGame/Example/*.cs ../MyGame/*.cs \ + FlatBuffersTestClassAttribute.cs FlatBuffersTestMethodAttribute.cs Assert.cs FlatBuffersExampleTests.cs Program.cs ByteBufferTests.cs FlatBufferBuilderTests.cs FlatBuffersFuzzTests.cs FuzzTestData.cs Lcg.cs TestTable.cs +mono --debug ./fbnettest.exe rm fbnettest.exe rm Resources/monsterdata_cstest.mon +rm Resources/monsterdata_cstest_sp.mon + +# Repeat with unsafe versions + +mcs -debug -out:./fbnettest.exe \ + -unsafe -d:UNSAFE_BYTEBUFFER \ + ../../net/FlatBuffers/*.cs ../MyGame/Example/*.cs ../MyGame/*.cs \ + FlatBuffersTestClassAttribute.cs FlatBuffersTestMethodAttribute.cs Assert.cs FlatBuffersExampleTests.cs Program.cs ByteBufferTests.cs FlatBufferBuilderTests.cs FlatBuffersFuzzTests.cs FuzzTestData.cs Lcg.cs TestTable.cs +mono --debug ./fbnettest.exe +rm fbnettest.exe +rm Resources/monsterdata_cstest.mon +rm Resources/monsterdata_cstest_sp.mon diff --git a/tests/JavaTest.java b/tests/JavaTest.java index 9a5b6763dde6afedd2f5d8e3b15a7ba68aeb52e9..a9dedf9790f68dca44a6f5db66ae2d57cd2ea1e9 100644 --- a/tests/JavaTest.java +++ b/tests/JavaTest.java @@ -21,6 +21,8 @@ import java.nio.channels.FileChannel; import MyGame.Example.*; import NamespaceA.*; import NamespaceA.NamespaceB.*; +import com.google.flatbuffers.ByteBufferUtil; +import static com.google.flatbuffers.Constants.*; import com.google.flatbuffers.FlatBufferBuilder; class JavaTest { @@ -53,7 +55,8 @@ class JavaTest { // better for performance. FlatBufferBuilder fbb = new FlatBufferBuilder(1); - TestBuilderBasics(fbb); + TestBuilderBasics(fbb, true); + TestBuilderBasics(fbb, false); TestExtendedBuffer(fbb.dataBuffer().asReadOnlyBuffer()); @@ -244,14 +247,14 @@ class JavaTest { FlatBufferBuilder fbb = new FlatBufferBuilder(1, new MappedByteBufferFactory()); - TestBuilderBasics(fbb); + TestBuilderBasics(fbb, false); } static void TestSizedInputStream() { // Test on default FlatBufferBuilder that uses HeapByteBuffer FlatBufferBuilder fbb = new FlatBufferBuilder(1); - TestBuilderBasics(fbb); + TestBuilderBasics(fbb, false); InputStream in = fbb.sizedInputStream(); byte[] array = fbb.sizedByteArray(); @@ -271,7 +274,7 @@ class JavaTest { TestEq(count, array.length); } - static void TestBuilderBasics(FlatBufferBuilder fbb) { + static void TestBuilderBasics(FlatBufferBuilder fbb, boolean sizePrefix) { int[] names = {fbb.createString("Frodo"), fbb.createString("Barney"), fbb.createString("Wilma")}; int[] off = new int[3]; Monster.startMonster(fbb); @@ -321,7 +324,11 @@ class JavaTest { Monster.addTestarrayoftables(fbb, sortMons); int mon = Monster.endMonster(fbb); - Monster.finishMonsterBuffer(fbb, mon); + if (sizePrefix) { + Monster.finishSizePrefixedMonsterBuffer(fbb, mon); + } else { + Monster.finishMonsterBuffer(fbb, mon); + } // Write the result to a file for debugging purposes: // Note that the binaries are not necessarily identical, since the JSON @@ -329,7 +336,8 @@ class JavaTest { // Java code. They are functionally equivalent though. try { - FileChannel fc = new FileOutputStream("monsterdata_java_wire.mon").getChannel(); + String filename = "monsterdata_java_wire" + (sizePrefix ? "_sp" : "") + ".mon"; + FileChannel fc = new FileOutputStream(filename).getChannel(); fc.write(fbb.dataBuffer().duplicate()); fc.close(); } catch(java.io.IOException e) { @@ -338,18 +346,24 @@ class JavaTest { } // Test it: - TestExtendedBuffer(fbb.dataBuffer()); + ByteBuffer dataBuffer = fbb.dataBuffer(); + if (sizePrefix) { + TestEq(ByteBufferUtil.getSizePrefix(dataBuffer) + SIZE_PREFIX_LENGTH, + dataBuffer.remaining()); + dataBuffer = ByteBufferUtil.removeSizePrefix(dataBuffer); + } + TestExtendedBuffer(dataBuffer); // Make sure it also works with read only ByteBuffers. This is slower, // since creating strings incurs an additional copy // (see Table.__string). - TestExtendedBuffer(fbb.dataBuffer().asReadOnlyBuffer()); + TestExtendedBuffer(dataBuffer.asReadOnlyBuffer()); TestEnums(); //Attempt to mutate Monster fields and check whether the buffer has been mutated properly // revert to original values after testing - Monster monster = Monster.getRootAsMonster(fbb.dataBuffer()); + Monster monster = Monster.getRootAsMonster(dataBuffer); // mana is optional and does not exist in the buffer so the mutation should fail // the mana field should retain its default value diff --git a/tests/MyGame/Example/Monster.cs b/tests/MyGame/Example/Monster.cs index c554c3cd1f953495eeb9bc80ae98fdee5bbaf3e0..b7844fb36f1589af63674b6f2dc8de327ccbacca 100644 --- a/tests/MyGame/Example/Monster.cs +++ b/tests/MyGame/Example/Monster.cs @@ -203,6 +203,7 @@ public struct Monster : IFlatbufferObject return new Offset<Monster>(o); } public static void FinishMonsterBuffer(FlatBufferBuilder builder, Offset<Monster> offset) { builder.Finish(offset.Value, "MONS"); } + public static void FinishSizePrefixedMonsterBuffer(FlatBufferBuilder builder, Offset<Monster> offset) { builder.FinishSizePrefixed(offset.Value, "MONS"); } public static VectorOffset CreateSortedVectorOfMonster(FlatBufferBuilder builder, Offset<Monster>[] offsets) { Array.Sort(offsets, (Offset<Monster> o1, Offset<Monster> o2) => Table.CompareStrings(Table.__offset(10, o1.Value, builder.DataBuffer), Table.__offset(10, o2.Value, builder.DataBuffer), builder.DataBuffer)); diff --git a/tests/MyGame/Example/Monster.java b/tests/MyGame/Example/Monster.java index 6048dc7b4e8f1dd0002b8d7aa45337dd0d9899fa..8a5bb0e7e0b1a339cc222d4f7075a1271b8f31d9 100644 --- a/tests/MyGame/Example/Monster.java +++ b/tests/MyGame/Example/Monster.java @@ -228,6 +228,7 @@ public final class Monster extends Table { return o; } public static void finishMonsterBuffer(FlatBufferBuilder builder, int offset) { builder.finish(offset, "MONS"); } + public static void finishSizePrefixedMonsterBuffer(FlatBufferBuilder builder, int offset) { builder.finishSizePrefixed(offset, "MONS"); } @Override protected int keysCompare(Integer o1, Integer o2, ByteBuffer _bb) { return compareStrings(__offset(10, o1, _bb), __offset(10, o2, _bb), _bb); } diff --git a/tests/monster_test.bfbs b/tests/monster_test.bfbs index 0093defa39236196ec7a5b46b46b6c25ac81c770..76057281704be584128772e75e929d6ede2867cb 100644 Binary files a/tests/monster_test.bfbs and b/tests/monster_test.bfbs differ diff --git a/tests/namespace_test/namespace_test2_generated.ts b/tests/namespace_test/namespace_test2_generated.ts index 61cf63123395b3347c008942c24dec08010068f8..aa623a87974332a59ed8d73e38abbeb982b0f110 100644 --- a/tests/namespace_test/namespace_test2_generated.ts +++ b/tests/namespace_test/namespace_test2_generated.ts @@ -1,6 +1,6 @@ // automatically generated by the FlatBuffers compiler, do not modify -import * as NS39599748 from "./namespace_test1_generated"; +import * as NS4989953370203581498 from "./namespace_test1_generated"; /** * @constructor */ @@ -39,24 +39,24 @@ static getRootAsTableInFirstNS(bb:flatbuffers.ByteBuffer, obj?:TableInFirstNS):T * @param {NamespaceA.NamespaceB.TableInNestedNS=} obj * @returns {NamespaceA.NamespaceB.TableInNestedNS|null} */ -fooTable(obj?:NS39599748.NamespaceA.NamespaceB.TableInNestedNS):NS39599748.NamespaceA.NamespaceB.TableInNestedNS|null { +fooTable(obj?:NS4989953370203581498.NamespaceA.NamespaceB.TableInNestedNS):NS4989953370203581498.NamespaceA.NamespaceB.TableInNestedNS|null { var offset = this.bb!.__offset(this.bb_pos, 4); - return offset ? (obj || new NS39599748.NamespaceA.NamespaceB.TableInNestedNS).__init(this.bb!.__indirect(this.bb_pos + offset), this.bb!) : null; + return offset ? (obj || new NS4989953370203581498.NamespaceA.NamespaceB.TableInNestedNS).__init(this.bb!.__indirect(this.bb_pos + offset), this.bb!) : null; }; /** * @returns {NamespaceA.NamespaceB.EnumInNestedNS} */ -fooEnum():NS39599748.NamespaceA.NamespaceB.EnumInNestedNS { +fooEnum():NS4989953370203581498.NamespaceA.NamespaceB.EnumInNestedNS { var offset = this.bb!.__offset(this.bb_pos, 6); - return offset ? /** @type {NamespaceA.NamespaceB.EnumInNestedNS} */ (this.bb!.readInt8(this.bb_pos + offset)) : NS39599748.NamespaceA.NamespaceB.EnumInNestedNS.A; + return offset ? /** @type {NamespaceA.NamespaceB.EnumInNestedNS} */ (this.bb!.readInt8(this.bb_pos + offset)) : NS4989953370203581498.NamespaceA.NamespaceB.EnumInNestedNS.A; }; /** * @param {NamespaceA.NamespaceB.EnumInNestedNS} value * @returns {boolean} */ -mutate_foo_enum(value:NS39599748.NamespaceA.NamespaceB.EnumInNestedNS):boolean { +mutate_foo_enum(value:NS4989953370203581498.NamespaceA.NamespaceB.EnumInNestedNS):boolean { var offset = this.bb!.__offset(this.bb_pos, 6); if (offset === 0) { @@ -71,9 +71,9 @@ mutate_foo_enum(value:NS39599748.NamespaceA.NamespaceB.EnumInNestedNS):boolean { * @param {NamespaceA.NamespaceB.StructInNestedNS=} obj * @returns {NamespaceA.NamespaceB.StructInNestedNS|null} */ -fooStruct(obj?:NS39599748.NamespaceA.NamespaceB.StructInNestedNS):NS39599748.NamespaceA.NamespaceB.StructInNestedNS|null { +fooStruct(obj?:NS4989953370203581498.NamespaceA.NamespaceB.StructInNestedNS):NS4989953370203581498.NamespaceA.NamespaceB.StructInNestedNS|null { var offset = this.bb!.__offset(this.bb_pos, 8); - return offset ? (obj || new NS39599748.NamespaceA.NamespaceB.StructInNestedNS).__init(this.bb_pos + offset, this.bb!) : null; + return offset ? (obj || new NS4989953370203581498.NamespaceA.NamespaceB.StructInNestedNS).__init(this.bb_pos + offset, this.bb!) : null; }; /** @@ -95,8 +95,8 @@ static addFooTable(builder:flatbuffers.Builder, fooTableOffset:flatbuffers.Offse * @param {flatbuffers.Builder} builder * @param {NamespaceA.NamespaceB.EnumInNestedNS} fooEnum */ -static addFooEnum(builder:flatbuffers.Builder, fooEnum:NS39599748.NamespaceA.NamespaceB.EnumInNestedNS) { - builder.addFieldInt8(1, fooEnum, NS39599748.NamespaceA.NamespaceB.EnumInNestedNS.A); +static addFooEnum(builder:flatbuffers.Builder, fooEnum:NS4989953370203581498.NamespaceA.NamespaceB.EnumInNestedNS) { + builder.addFieldInt8(1, fooEnum, NS4989953370203581498.NamespaceA.NamespaceB.EnumInNestedNS.A); }; /** diff --git a/tests/py_test.py b/tests/py_test.py index aeef729af6a998cd631e1bc8727793d6fe62c5ab..1b76b61a1578f49c8cb985cc73922d410409d002 100644 --- a/tests/py_test.py +++ b/tests/py_test.py @@ -25,6 +25,7 @@ import unittest from flatbuffers import compat +from flatbuffers import util from flatbuffers.compat import range_func as compat_range from flatbuffers.compat import NumpyRequiredForThisFeature @@ -56,9 +57,11 @@ def assertRaises(test_case, fn, exception_class): class TestWireFormat(unittest.TestCase): def test_wire_format(self): # Verify that using the generated Python code builds a buffer without - # returning errors, and is interpreted correctly: - gen_buf, gen_off = make_monster_from_generated_code() - CheckReadBuffer(gen_buf, gen_off) + # returning errors, and is interpreted correctly, for size prefixed + # representation and regular: + for sizePrefix in [True, False]: + gen_buf, gen_off = make_monster_from_generated_code(sizePrefix = sizePrefix) + CheckReadBuffer(gen_buf, gen_off, sizePrefix = sizePrefix) # Verify that the canonical flatbuffer file is readable by the # generated Python code. Note that context managers are not part of @@ -74,7 +77,7 @@ class TestWireFormat(unittest.TestCase): f.close() -def CheckReadBuffer(buf, offset): +def CheckReadBuffer(buf, offset, sizePrefix = False): ''' CheckReadBuffer checks that the given buffer is evaluated correctly as the example Monster. ''' @@ -83,6 +86,11 @@ def CheckReadBuffer(buf, offset): if not stmt: raise AssertionError('CheckReadBuffer case failed') + if sizePrefix: + size = util.GetSizePrefix(buf, offset) + # taken from the size of monsterdata_python_wire.mon, minus 4 + asserter(size == 348) + buf, offset = util.RemoveSizePrefix(buf, offset) monster = MyGame.Example.Monster.Monster.GetRootAsMonster(buf, offset) asserter(monster.Hp() == 80) @@ -810,7 +818,7 @@ class TestByteLayout(unittest.TestCase): ]) -def make_monster_from_generated_code(): +def make_monster_from_generated_code(sizePrefix = False): ''' Use generated code to build the example Monster. ''' b = flatbuffers.Builder(0) @@ -871,7 +879,10 @@ def make_monster_from_generated_code(): MyGame.Example.Monster.MonsterAddVectorOfDoubles(b, VectorOfDoubles) mon = MyGame.Example.Monster.MonsterEnd(b) - b.Finish(mon) + if sizePrefix: + b.FinishSizePrefixed(mon) + else: + b.Finish(mon) return b.Bytes, b.Head()