Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

implement binary packed repeated encoding #145

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

implement binary packed repeated encoding #145

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 22 additions & 1 deletion src/Falanx.Proto.Codec.Binary/Codec.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ namespace Falanx.Proto.Codec.Binary
type proto_map<'Key, 'Value> = Collections.Generic.Dictionary<'Key, 'Value>

type Writer<'T> = FieldNum -> ZeroCopyBuffer -> 'T -> unit
type PackedWriter<'T> = (ResizeArray<'T> -> uint64, 'T -> ZeroCopyBuffer -> unit) // (obtain len fn, encode val fn)
type Reader<'T> = RawField -> 'T

// Eventually this interface could be addd to Froto itself
Expand Down Expand Up @@ -89,6 +90,20 @@ namespace Falanx.Proto.Codec.Binary
let writeBool: Writer<proto_bool> = write Encode.fromBool
let writeString: Writer<proto_string> = write Encode.fromString
let writeBytes: Writer<proto_bytes> = write Encode.fromBytes

let writeDoublePacked: PackedWriter<proto_double> = (fun a -> a.length * 8, Pack.toDouble)
let writeFloatPacked: PackedWriter<proto_float> = (fun a -> a.length * 4, Pack.toSingle)
let writeInt32Packed: PackedWriter<proto_int32> = (Seq.sumBy (uint64 >> Utility.varIntLenNoDefault), uint64 >> Pack.toVarint)
let writeInt64Packed: PackedWriter<proto_int64> = (Seq.sumBy (uint64 >> Utility.varIntLenNoDefault), uint64 >> Pack.toVarint)
let writeUInt32Packed: PackedWriter<proto_uint32> = (Seq.sumBy (uint64 >> Utility.varIntLenNoDefault), uint64 >> Pack.toVarint)
let writeUInt64Packed: PackedWriter<proto_uint64> = (Seq.sumBy (uint64 >> Utility.varIntLenNoDefault), Pack.toVarint)
let writeSInt32Packed: PackedWriter<proto_sint32> = (Seq.sumBy (zigZag32 >> uint64 >> Utility.varIntLenNoDefault), zigZag32 >> uint64 >> Pack.toVarint)
let writeSInt64Packed: PackedWriter<proto_sint64> = (Seq.sumBy (zigZag64 >> uint64 >> Utility.varIntLenNoDefault), zigZag32 >> uint64 >> Pack.toVarint)
let writeFixed32Packed: PackedWriter<proto_fixed32> = (fun a -> a.length * 4, Pack.toFixed32)
let writeFixed64Packed: PackedWriter<proto_fixed64> = (fun a -> a.length * 8, Pack.toFixed64)
let writeSFixed32Packed: PackedWriter<proto_sfixed32> = (fun a -> a.length * 4, uint32 >> Pack.toFixed32)
let writeSFixed64Packed: PackedWriter<proto_sfixed64> = (fun a -> a.length * 8, uint64 >> Pack.toFixed64)
let writeBoolPacked: PackedWriter<proto_bool> = (fun a -> a.length, Pack.toBool)

/// Serializes optional field using provided function to handle inner value if present
let writeOptional (writeInner: Writer<'T>) (position: FieldNum) buffer value =
Expand Down Expand Up @@ -119,7 +134,13 @@ namespace Falanx.Proto.Codec.Binary

let writeRepeated (writeItem: Writer<'T>) position buffer value =
value |> Seq.iter (writeItem position buffer)


let writeRepeatedPacked ((getLen, writeItemPacked): PackedWriter<'T>) position buffer (value: ResizeArray<'T>) =
let vlen = getLen value
Pack.toTag position WireType.LengthDelimited buffer |> ignore
Pack.toVarint vlen buffer |> ignore
Seq.iter (fun v -> writeItemPacked v buffer) value

let writeRepeatedEmbedded<'a when 'a :> IMessage > (position, buffer, value ) =
for (v: 'a) in (value: ResizeArray<'a>) do
buffer
Expand Down
33 changes: 27 additions & 6 deletions src/Falanx.Proto.Core/BinaryCodec/Serialization.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,23 @@ module Serialization =
| "bool" -> <@@ writeBool @@>
| "string" -> <@@ writeString @@>
| "bytes" -> <@@ writeBytes @@>
| x -> failwithf "Type %A nor supported" x
| x -> failwithf "Type %A not supported" x

let primitivePackedWriter = function
| "double" -> <@@ writeDoublePacked @@>
| "float" -> <@@ writeFloatPacked @@>
| "int32" -> <@@ writeInt32Packed @@>
| "int64" -> <@@ writeInt64Packed @@>
| "uint32" -> <@@ writeUInt32Packed @@>
| "uint64" -> <@@ writeUInt64Packed @@>
| "sint32" -> <@@ writeSInt32Packed @@>
| "sint64" -> <@@ writeSInt64Packed @@>
| "fixed32" -> <@@ writeFixed32Packed @@>
| "fixed64" -> <@@ writeFixed64Packed @@>
| "sfixed32" -> <@@ writeSFixed32Packed @@>
| "sfixed64" -> <@@ writeSFixed64Packed @@>
| "bool" -> <@@ writeBoolPacked @@>
| x -> failwithf "Type %A not supported" x

let serializeMapExpr buffer this (map: MapDescriptor) =
let keyWriter = primitiveWriter map.KeyType.ProtobufType
Expand All @@ -54,20 +70,25 @@ module Serialization =
[keyWriter; positionExpr; buffer; mapExpr]
<@@ writeEnumMap x x x x @@>

let callPrimitive writer (prop: PropertyDescriptor) rule position buffer value =
let callPrimitive (prop: PropertyDescriptor) rule position buffer value =
let args = [position; buffer; value]
match rule with
| Required ->
Expr.apply writer args
Expr.apply (primitiveWriter prop.Type.ProtobufType) args
| Optional ->
Expr.callStaticGeneric
[prop.Type.UnderlyingType]
[writer; position; buffer; value]
[primitiveWriter prop.Type.ProtobufType; position; buffer; value]
<@@ writeOption x x x x @@>
| Repeated when prop.Type.ProtobufType <> "string" && prop.Type.ProtobufType <> "bytes" ->
Expr.callStaticGeneric
[prop.Type.UnderlyingType]
((primitivePackedWriter prop.Type.ProtobufType) ::args)
<@@ writeRepeatedPacked x x x x @@>
| Repeated ->
Expr.callStaticGeneric
[prop.Type.UnderlyingType]
(writer::args)
((primitiveWriter prop.Type.ProtobufType) ::args)
<@@ writeRepeated x x x x @@>


Expand Down Expand Up @@ -114,7 +135,7 @@ module Serialization =
[ <@@ writeInt32 @@> ;position; buffer; seqMap]
<@@ writeRepeated x x x x @@>
| Primitive _type, rule ->
callPrimitive (primitiveWriter prop.Type.ProtobufType) prop rule position buffer value
callPrimitive prop rule position buffer value
with
| ex ->
printfn "Failed to serialize property %s: %O. Error: %O" prop.ProvidedProperty.Name value.Type ex
Expand Down