diff options
Diffstat (limited to 'ruby')
22 files changed, 545 insertions, 100 deletions
diff --git a/ruby/Rakefile b/ruby/Rakefile index fa29c315..ba1cf4cf 100644 --- a/ruby/Rakefile +++ b/ruby/Rakefile @@ -31,7 +31,7 @@ genproto_output = [] unless ENV['IN_DOCKER'] == 'true' well_known_protos.each do |proto_file| input_file = "../src/" + proto_file - output_file = "lib/" + proto_file.sub(/\.proto$/, ".rb") + output_file = "lib/" + proto_file.sub(/\.proto$/, "_pb.rb") genproto_output << output_file file output_file => input_file do |file_task| sh "../src/protoc -I../src --ruby_out=lib #{input_file}" @@ -80,10 +80,15 @@ end # Proto for tests. genproto_output << "tests/generated_code.rb" +genproto_output << "tests/test_import.rb" file "tests/generated_code.rb" => "tests/generated_code.proto" do |file_task| sh "../src/protoc --ruby_out=. tests/generated_code.proto" end +file "tests/test_import.rb" => "tests/test_import.proto" do |file_task| + sh "../src/protoc --ruby_out=. tests/test_import.proto" +end + task :genproto => genproto_output task :clean do diff --git a/ruby/ext/google/protobuf_c/encode_decode.c b/ruby/ext/google/protobuf_c/encode_decode.c index f6bea50f..08c72bcc 100644 --- a/ruby/ext/google/protobuf_c/encode_decode.c +++ b/ruby/ext/google/protobuf_c/encode_decode.c @@ -54,7 +54,7 @@ VALUE noleak_rb_str_cat(VALUE rb_str, const char *str, long len) { static const void* newhandlerdata(upb_handlers* h, uint32_t ofs) { size_t* hd_ofs = ALLOC(size_t); *hd_ofs = ofs; - upb_handlers_addcleanup(h, hd_ofs, free); + upb_handlers_addcleanup(h, hd_ofs, xfree); return hd_ofs; } @@ -69,7 +69,7 @@ static const void *newsubmsghandlerdata(upb_handlers* h, uint32_t ofs, submsg_handlerdata_t *hd = ALLOC(submsg_handlerdata_t); hd->ofs = ofs; hd->md = upb_fielddef_msgsubdef(f); - upb_handlers_addcleanup(h, hd, free); + upb_handlers_addcleanup(h, hd, xfree); return hd; } @@ -99,7 +99,7 @@ static const void *newoneofhandlerdata(upb_handlers *h, } else { hd->md = NULL; } - upb_handlers_addcleanup(h, hd, free); + upb_handlers_addcleanup(h, hd, xfree); return hd; } @@ -135,7 +135,7 @@ static void* appendstr_handler(void *closure, VALUE ary = (VALUE)closure; VALUE str = rb_str_new2(""); rb_enc_associate(str, kRubyStringUtf8Encoding); - RepeatedField_push(ary, str); + RepeatedField_push_native(ary, &str); return (void*)str; } @@ -146,7 +146,7 @@ static void* appendbytes_handler(void *closure, VALUE ary = (VALUE)closure; VALUE str = rb_str_new2(""); rb_enc_associate(str, kRubyString8bitEncoding); - RepeatedField_push(ary, str); + RepeatedField_push_native(ary, &str); return (void*)str; } @@ -182,6 +182,23 @@ static size_t stringdata_handler(void* closure, const void* hd, return len; } +static bool stringdata_end_handler(void* closure, const void* hd) { + MessageHeader* msg = closure; + const size_t *ofs = hd; + VALUE rb_str = DEREF(msg, *ofs, VALUE); + rb_obj_freeze(rb_str); + return true; +} + +static bool appendstring_end_handler(void* closure, const void* hd) { + VALUE ary = (VALUE)closure; + int size = RepeatedField_size(ary); + VALUE* last = RepeatedField_index_native(ary, size - 1); + VALUE rb_str = *last; + rb_obj_freeze(rb_str); + return true; +} + // Appends a submessage to a repeated field (a regular Ruby array for now). static void *appendsubmsg_handler(void *closure, const void *hd) { VALUE ary = (VALUE)closure; @@ -238,10 +255,54 @@ typedef struct { // value into the map. typedef struct { VALUE map; + const map_handlerdata_t* handlerdata; char key_storage[NATIVE_SLOT_MAX_SIZE]; char value_storage[NATIVE_SLOT_MAX_SIZE]; } map_parse_frame_t; +static void MapParseFrame_mark(void* _self) { + map_parse_frame_t* frame = _self; + + // This shouldn't strictly be necessary since this should be rooted by the + // message itself, but it can't hurt. + rb_gc_mark(frame->map); + + native_slot_mark(frame->handlerdata->key_field_type, &frame->key_storage); + native_slot_mark(frame->handlerdata->value_field_type, &frame->value_storage); +} + +void MapParseFrame_free(void* self) { + xfree(self); +} + +rb_data_type_t MapParseFrame_type = { + "MapParseFrame", + { MapParseFrame_mark, MapParseFrame_free, NULL }, +}; + +// Array of Ruby objects wrapping map_parse_frame_t. +// We don't allow multiple concurrent decodes, so we assume that this global +// variable is specific to the "current" decode. +VALUE map_parse_frames; + +static map_parse_frame_t* map_push_frame(VALUE map, + const map_handlerdata_t* handlerdata) { + map_parse_frame_t* frame = ALLOC(map_parse_frame_t); + frame->handlerdata = handlerdata; + frame->map = map; + native_slot_init(handlerdata->key_field_type, &frame->key_storage); + native_slot_init(handlerdata->value_field_type, &frame->value_storage); + + rb_ary_push(map_parse_frames, + TypedData_Wrap_Struct(rb_cObject, &MapParseFrame_type, frame)); + + return frame; +} + +static void map_pop_frame() { + rb_ary_pop(map_parse_frames); +} + // Handler to begin a map entry: allocates a temporary frame. This is the // 'startsubmsg' handler on the msgdef that contains the map field. static void *startmapentry_handler(void *closure, const void *hd) { @@ -249,13 +310,7 @@ static void *startmapentry_handler(void *closure, const void *hd) { const map_handlerdata_t* mapdata = hd; VALUE map_rb = DEREF(msg, mapdata->ofs, VALUE); - map_parse_frame_t* frame = ALLOC(map_parse_frame_t); - frame->map = map_rb; - - native_slot_init(mapdata->key_field_type, &frame->key_storage); - native_slot_init(mapdata->value_field_type, &frame->value_storage); - - return frame; + return map_push_frame(map_rb, mapdata); } // Handler to end a map entry: inserts the value defined during the message into @@ -281,7 +336,7 @@ static bool endmap_handler(void *closure, const void *hd, upb_status* s) { &frame->value_storage); Map_index_set(frame->map, key, value); - free(frame); + map_pop_frame(); return true; } @@ -360,6 +415,13 @@ static void *oneofbytes_handler(void *closure, return (void*)str; } +static bool oneofstring_end_handler(void* closure, const void* hd) { + MessageHeader* msg = closure; + const oneof_handlerdata_t *oneofdata = hd; + rb_obj_freeze(DEREF(msg, oneofdata->ofs, VALUE)); + return true; +} + // Handler for a submessage field in a oneof. static void *oneofsubmsg_handler(void *closure, const void *hd) { @@ -426,6 +488,7 @@ static void add_handlers_for_repeated_field(upb_handlers *h, appendbytes_handler : appendstr_handler, NULL); upb_handlers_setstring(h, f, stringdata_handler, NULL); + upb_handlers_setendstr(h, f, appendstring_end_handler, NULL); break; } case UPB_TYPE_MESSAGE: { @@ -462,6 +525,7 @@ static void add_handlers_for_singular_field(upb_handlers *h, is_bytes ? bytes_handler : str_handler, &attr); upb_handlers_setstring(h, f, stringdata_handler, &attr); + upb_handlers_setendstr(h, f, stringdata_end_handler, &attr); upb_handlerattr_uninit(&attr); break; } @@ -484,7 +548,7 @@ static void add_handlers_for_mapfield(upb_handlers* h, map_handlerdata_t* hd = new_map_handlerdata(offset, map_msgdef, desc); upb_handlerattr attr = UPB_HANDLERATTR_INITIALIZER; - upb_handlers_addcleanup(h, hd, free); + upb_handlers_addcleanup(h, hd, xfree); upb_handlerattr_sethandlerdata(&attr, hd); upb_handlers_setstartsubmsg(h, fielddef, startmapentry_handler, &attr); upb_handlerattr_uninit(&attr); @@ -499,7 +563,7 @@ static void add_handlers_for_mapentry(const upb_msgdef* msgdef, map_handlerdata_t* hd = new_map_handlerdata(0, msgdef, desc); upb_handlerattr attr = UPB_HANDLERATTR_INITIALIZER; - upb_handlers_addcleanup(h, hd, free); + upb_handlers_addcleanup(h, hd, xfree); upb_handlerattr_sethandlerdata(&attr, hd); upb_handlers_setendmsg(h, endmap_handler, &attr); @@ -546,6 +610,7 @@ static void add_handlers_for_oneof_field(upb_handlers *h, oneofbytes_handler : oneofstr_handler, &attr); upb_handlers_setstring(h, f, stringdata_handler, NULL); + upb_handlers_setendstr(h, f, oneofstring_end_handler, &attr); break; } case UPB_TYPE_MESSAGE: { @@ -710,6 +775,10 @@ VALUE Message_decode(VALUE klass, VALUE data) { msg_rb = rb_class_new_instance(0, NULL, msgklass); TypedData_Get_Struct(msg_rb, MessageHeader, &Message_type, msg); + // We generally expect this to be clear already, but clear it in case parsing + // previously got interrupted somehow. + rb_ary_clear(map_parse_frames); + { const upb_pbdecodermethod* method = msgdef_decodermethod(desc); const upb_handlers* h = upb_pbdecodermethod_desthandlers(method); @@ -754,6 +823,10 @@ VALUE Message_decode_json(VALUE klass, VALUE data) { msg_rb = rb_class_new_instance(0, NULL, msgklass); TypedData_Get_Struct(msg_rb, MessageHeader, &Message_type, msg); + // We generally expect this to be clear already, but clear it in case parsing + // previously got interrupted somehow. + rb_ary_clear(map_parse_frames); + { const upb_json_parsermethod* method = msgdef_jsonparsermethod(desc); stackenv se; @@ -863,9 +936,13 @@ static void putstr(VALUE str, const upb_fielddef *f, upb_sink *sink) { assert(BUILTIN_TYPE(str) == RUBY_T_STRING); - // Ensure that the string has the correct encoding. We also check at field-set - // time, but the user may have mutated the string object since then. - native_slot_validate_string_encoding(upb_fielddef_type(f), str); + // We should be guaranteed that the string has the correct encoding because + // we ensured this at assignment time and then froze the string. + if (upb_fielddef_type(f) == UPB_TYPE_STRING) { + assert(rb_enc_from_index(ENCODING_GET(value)) == kRubyStringUtf8Encoding); + } else { + assert(rb_enc_from_index(ENCODING_GET(value)) == kRubyString8bitEncoding); + } upb_sink_startstr(sink, getsel(f, UPB_HANDLER_STARTSTR), RSTRING_LEN(str), &subsink); diff --git a/ruby/ext/google/protobuf_c/map.c b/ruby/ext/google/protobuf_c/map.c index 92fc7286..12f1f9dd 100644 --- a/ruby/ext/google/protobuf_c/map.c +++ b/ruby/ext/google/protobuf_c/map.c @@ -63,16 +63,16 @@ // construct a key byte sequence if needed. |out_key| and |out_length| provide // the resulting key data/length. #define TABLE_KEY_BUF_LENGTH 8 // sizeof(uint64_t) -static void table_key(Map* self, VALUE key, - char* buf, - const char** out_key, - size_t* out_length) { +static VALUE table_key(Map* self, VALUE key, + char* buf, + const char** out_key, + size_t* out_length) { switch (self->key_type) { case UPB_TYPE_BYTES: case UPB_TYPE_STRING: // Strings: use string content directly. Check_Type(key, T_STRING); - native_slot_validate_string_encoding(self->key_type, key); + key = native_slot_encode_and_freeze_string(self->key_type, key); *out_key = RSTRING_PTR(key); *out_length = RSTRING_LEN(key); break; @@ -93,6 +93,8 @@ static void table_key(Map* self, VALUE key, assert(false); break; } + + return key; } static VALUE table_key_to_ruby(Map* self, const char* buf, size_t length) { @@ -357,7 +359,7 @@ VALUE Map_index(VALUE _self, VALUE key) { const char* keyval = NULL; size_t length = 0; upb_value v; - table_key(self, key, keybuf, &keyval, &length); + key = table_key(self, key, keybuf, &keyval, &length); if (upb_strtable_lookup2(&self->table, keyval, length, &v)) { void* mem = value_memory(&v); @@ -383,7 +385,7 @@ VALUE Map_index_set(VALUE _self, VALUE key, VALUE value) { size_t length = 0; upb_value v; void* mem; - table_key(self, key, keybuf, &keyval, &length); + key = table_key(self, key, keybuf, &keyval, &length); mem = value_memory(&v); native_slot_set(self->value_type, self->value_type_class, mem, value); @@ -411,7 +413,7 @@ VALUE Map_has_key(VALUE _self, VALUE key) { char keybuf[TABLE_KEY_BUF_LENGTH]; const char* keyval = NULL; size_t length = 0; - table_key(self, key, keybuf, &keyval, &length); + key = table_key(self, key, keybuf, &keyval, &length); if (upb_strtable_lookup2(&self->table, keyval, length, NULL)) { return Qtrue; @@ -434,7 +436,7 @@ VALUE Map_delete(VALUE _self, VALUE key) { const char* keyval = NULL; size_t length = 0; upb_value v; - table_key(self, key, keybuf, &keyval, &length); + key = table_key(self, key, keybuf, &keyval, &length); if (upb_strtable_remove2(&self->table, keyval, length, &v)) { void* mem = value_memory(&v); diff --git a/ruby/ext/google/protobuf_c/protobuf.c b/ruby/ext/google/protobuf_c/protobuf.c index 7cde4aec..98963667 100644 --- a/ruby/ext/google/protobuf_c/protobuf.c +++ b/ruby/ext/google/protobuf_c/protobuf.c @@ -112,4 +112,6 @@ void Init_protobuf_c() { upb_def_to_ruby_obj_map = rb_hash_new(); rb_gc_register_address(&upb_def_to_ruby_obj_map); + map_parse_frames = rb_ary_new(); + rb_gc_register_address(&map_parse_frames); } diff --git a/ruby/ext/google/protobuf_c/protobuf.h b/ruby/ext/google/protobuf_c/protobuf.h index 2834c894..d5ced567 100644 --- a/ruby/ext/google/protobuf_c/protobuf.h +++ b/ruby/ext/google/protobuf_c/protobuf.h @@ -166,6 +166,8 @@ extern VALUE cBuilder; extern VALUE cError; extern VALUE cParseError; +extern VALUE map_parse_frames; + // We forward-declare all of the Ruby method implementations here because we // sometimes call the methods directly across .c files, rather than going // through Ruby's method dispatching (e.g. during message parse). It's cleaner @@ -313,7 +315,7 @@ void native_slot_dup(upb_fieldtype_t type, void* to, void* from); void native_slot_deep_copy(upb_fieldtype_t type, void* to, void* from); bool native_slot_eq(upb_fieldtype_t type, void* mem1, void* mem2); -void native_slot_validate_string_encoding(upb_fieldtype_t type, VALUE value); +VALUE native_slot_encode_and_freeze_string(upb_fieldtype_t type, VALUE value); void native_slot_check_int_range_precision(upb_fieldtype_t type, VALUE value); extern rb_encoding* kRubyStringUtf8Encoding; @@ -366,6 +368,7 @@ RepeatedField* ruby_to_RepeatedField(VALUE value); VALUE RepeatedField_each(VALUE _self); VALUE RepeatedField_index(int argc, VALUE* argv, VALUE _self); void* RepeatedField_index_native(VALUE _self, int index); +int RepeatedField_size(VALUE _self); VALUE RepeatedField_index_set(VALUE _self, VALUE _index, VALUE val); void RepeatedField_reserve(RepeatedField* self, int new_size); VALUE RepeatedField_push(VALUE _self, VALUE val); diff --git a/ruby/ext/google/protobuf_c/repeated_field.c b/ruby/ext/google/protobuf_c/repeated_field.c index 83afbc91..47c207a5 100644 --- a/ruby/ext/google/protobuf_c/repeated_field.c +++ b/ruby/ext/google/protobuf_c/repeated_field.c @@ -244,6 +244,11 @@ void* RepeatedField_index_native(VALUE _self, int index) { return RepeatedField_memoryat(self, index, element_size); } +int RepeatedField_size(VALUE _self) { + RepeatedField* self = ruby_to_RepeatedField(_self); + return self->size; +} + /* * Private ruby method, used by RepeatedField.pop */ diff --git a/ruby/ext/google/protobuf_c/storage.c b/ruby/ext/google/protobuf_c/storage.c index 1c839781..3ff2bda6 100644 --- a/ruby/ext/google/protobuf_c/storage.c +++ b/ruby/ext/google/protobuf_c/storage.c @@ -117,25 +117,24 @@ void native_slot_check_int_range_precision(upb_fieldtype_t type, VALUE val) { } } -void native_slot_validate_string_encoding(upb_fieldtype_t type, VALUE value) { - bool bad_encoding = false; - rb_encoding* string_encoding = rb_enc_from_index(ENCODING_GET(value)); - if (type == UPB_TYPE_STRING) { - bad_encoding = - string_encoding != kRubyStringUtf8Encoding && - string_encoding != kRubyStringASCIIEncoding; - } else { - bad_encoding = - string_encoding != kRubyString8bitEncoding; - } - // Check that encoding is UTF-8 or ASCII (for string fields) or ASCII-8BIT - // (for bytes fields). - if (bad_encoding) { - rb_raise(rb_eTypeError, "Encoding for '%s' fields must be %s (was %s)", - (type == UPB_TYPE_STRING) ? "string" : "bytes", - (type == UPB_TYPE_STRING) ? "UTF-8 or ASCII" : "ASCII-8BIT", - rb_enc_name(string_encoding)); +VALUE native_slot_encode_and_freeze_string(upb_fieldtype_t type, VALUE value) { + rb_encoding* desired_encoding = (type == UPB_TYPE_STRING) ? + kRubyStringUtf8Encoding : kRubyString8bitEncoding; + VALUE desired_encoding_value = rb_enc_from_encoding(desired_encoding); + + // Note: this will not duplicate underlying string data unless necessary. + value = rb_str_encode(value, desired_encoding_value, 0, Qnil); + + if (type == UPB_TYPE_STRING && + rb_enc_str_coderange(value) == ENC_CODERANGE_BROKEN) { + rb_raise(rb_eEncodingError, "String is invalid UTF-8"); } + + // Ensure the data remains valid. Since we called #encode a moment ago, + // this does not freeze the string the user assigned. + rb_obj_freeze(value); + + return value; } void native_slot_set(upb_fieldtype_t type, VALUE type_class, @@ -181,8 +180,8 @@ void native_slot_set_value_and_case(upb_fieldtype_t type, VALUE type_class, if (CLASS_OF(value) != rb_cString) { rb_raise(rb_eTypeError, "Invalid argument for string field."); } - native_slot_validate_string_encoding(type, value); - DEREF(memory, VALUE) = value; + + DEREF(memory, VALUE) = native_slot_encode_and_freeze_string(type, value); break; } case UPB_TYPE_MESSAGE: { diff --git a/ruby/ext/google/protobuf_c/upb.c b/ruby/ext/google/protobuf_c/upb.c index 74a2a1db..544ebc04 100644 --- a/ruby/ext/google/protobuf_c/upb.c +++ b/ruby/ext/google/protobuf_c/upb.c @@ -11076,8 +11076,8 @@ static bool end_stringval(upb_json_parser *p) { case UPB_TYPE_STRING: { upb_selector_t sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSTR); - upb_sink_endstr(&p->top->sink, sel); p->top--; + upb_sink_endstr(&p->top->sink, sel); break; } @@ -11175,7 +11175,7 @@ static bool parse_mapentry_key(upb_json_parser *p) { sel = getsel_for_handlertype(p, UPB_HANDLER_STRING); upb_sink_putstring(&subsink, sel, buf, len, NULL); sel = getsel_for_handlertype(p, UPB_HANDLER_ENDSTR); - upb_sink_endstr(&subsink, sel); + upb_sink_endstr(&p->top->sink, sel); multipart_end(p); break; } diff --git a/ruby/google-protobuf.gemspec b/ruby/google-protobuf.gemspec index c542abf6..286d8fe3 100644 --- a/ruby/google-protobuf.gemspec +++ b/ruby/google-protobuf.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = "google-protobuf" - s.version = "3.0.0.alpha.6.0.0" + s.version = "3.0.0" s.licenses = ["BSD"] s.summary = "Protocol Buffers" s.description = "Protocol Buffers are Google's data interchange format." diff --git a/ruby/lib/google/protobuf.rb b/ruby/lib/google/protobuf.rb index 62bdd1bf..9b8d8231 100644 --- a/ruby/lib/google/protobuf.rb +++ b/ruby/lib/google/protobuf.rb @@ -45,7 +45,7 @@ if RUBY_PLATFORM == "java" require 'google/protobuf_java' else begin - require "google/#{RUBY_VERSION.sub(/\.\d$/, '')}/protobuf_c" + require "google/#{RUBY_VERSION.sub(/\.\d+$/, '')}/protobuf_c" rescue LoadError require 'google/protobuf_c' end diff --git a/ruby/lib/google/protobuf/well_known_types.rb b/ruby/lib/google/protobuf/well_known_types.rb new file mode 100644 index 00000000..547de874 --- /dev/null +++ b/ruby/lib/google/protobuf/well_known_types.rb @@ -0,0 +1,212 @@ +#!/usr/bin/ruby +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +require 'google/protobuf/any_pb' +require 'google/protobuf/duration_pb' +require 'google/protobuf/field_mask_pb' +require 'google/protobuf/struct_pb' +require 'google/protobuf/timestamp_pb' + +module Google + module Protobuf + + Any.class_eval do + def pack(msg, type_url_prefix='type.googleapis.com/') + if type_url_prefix.empty? or type_url_prefix[-1] != '/' then + self.type_url = "#{type_url_prefix}/#{msg.class.descriptor.name}" + else + self.type_url = "#{type_url_prefix}#{msg.class.descriptor.name}" + end + self.value = msg.to_proto + end + + def unpack(klass) + if self.is(klass) then + klass.decode(self.value) + else + nil + end + end + + def type_name + return self.type_url.split("/")[-1] + end + + def is(klass) + return self.type_name == klass.descriptor.name + end + end + + Timestamp.class_eval do + def to_time + Time.at(self.to_f) + end + + def from_time(time) + self.seconds = time.to_i + self.nanos = time.nsec + end + + def to_i + self.seconds + end + + def to_f + self.seconds + (self.nanos.to_f / 1_000_000_000) + end + end + + Duration.class_eval do + def to_f + self.seconds + (self.nanos.to_f / 1_000_000_000) + end + end + + class UnexpectedStructType < Google::Protobuf::Error; end + + Value.class_eval do + def to_ruby(recursive = false) + case self.kind + when :struct_value + if recursive + self.struct_value.to_h + else + self.struct_value + end + when :list_value + if recursive + self.list_value.to_a + else + self.list_value + end + when :null_value + nil + when :number_value + self.number_value + when :string_value + self.string_value + when :bool_value + self.bool_value + else + raise UnexpectedStructType + end + end + + def from_ruby(value) + case value + when NilClass + self.null_value = 0 + when Numeric + self.number_value = value + when String + self.string_value = value + when TrueClass + self.bool_value = true + when FalseClass + self.bool_value = false + when Struct + self.struct_value = value + when Hash + self.struct_value = Struct.from_hash(value) + when ListValue + self.list_value = value + when Array + self.list_value = ListValue.from_a(value) + else + raise UnexpectedStructType + end + end + end + + Struct.class_eval do + def [](key) + self.fields[key].to_ruby + end + + def []=(key, value) + unless key.is_a?(String) + raise UnexpectedStructType, "Struct keys must be strings." + end + self.fields[key] ||= Google::Protobuf::Value.new + self.fields[key].from_ruby(value) + end + + def to_h + ret = {} + self.fields.each { |key, val| ret[key] = val.to_ruby(true) } + ret + end + + def self.from_hash(hash) + ret = Struct.new + hash.each { |key, val| ret[key] = val } + ret + end + end + + ListValue.class_eval do + include Enumerable + + def length + self.values.length + end + + def [](index) + self.values[index].to_ruby + end + + def []=(index, value) + self.values[index].from_ruby(value) + end + + def <<(value) + wrapper = Google::Protobuf::Value.new + wrapper.from_ruby(value) + self.values << wrapper + end + + def each + self.values.each { |x| yield(x.to_ruby) } + end + + def to_a + self.values.map { |x| x.to_ruby(true) } + end + + def self.from_a(arr) + ret = ListValue.new + arr.each { |val| ret << val } + ret + end + end + + end +end diff --git a/ruby/pom.xml b/ruby/pom.xml index 4cbd6d30..99e8449b 100644 --- a/ruby/pom.xml +++ b/ruby/pom.xml @@ -86,7 +86,7 @@ <dependency> <groupId>com.google.protobuf</groupId> <artifactId>protobuf-java</artifactId> - <version>3.0.0-alpha-3</version> + <version>3.0.0-beta-4</version> </dependency> </dependencies> </project> diff --git a/ruby/src/main/java/com/google/protobuf/jruby/RubyMap.java b/ruby/src/main/java/com/google/protobuf/jruby/RubyMap.java index 2d4c03b5..3adaa2a8 100644 --- a/ruby/src/main/java/com/google/protobuf/jruby/RubyMap.java +++ b/ruby/src/main/java/com/google/protobuf/jruby/RubyMap.java @@ -148,8 +148,8 @@ public class RubyMap extends RubyObject { */ @JRubyMethod(name = "[]=") public IRubyObject indexSet(ThreadContext context, IRubyObject key, IRubyObject value) { - Utils.checkType(context, keyType, key, (RubyModule) valueTypeClass); - Utils.checkType(context, valueType, value, (RubyModule) valueTypeClass); + key = Utils.checkType(context, keyType, key, (RubyModule) valueTypeClass); + value = Utils.checkType(context, valueType, value, (RubyModule) valueTypeClass); IRubyObject symbol; if (valueType == Descriptors.FieldDescriptor.Type.ENUM && Utils.isRubyNum(value) && diff --git a/ruby/src/main/java/com/google/protobuf/jruby/RubyMessage.java b/ruby/src/main/java/com/google/protobuf/jruby/RubyMessage.java index 12893f73..462f8a69 100644 --- a/ruby/src/main/java/com/google/protobuf/jruby/RubyMessage.java +++ b/ruby/src/main/java/com/google/protobuf/jruby/RubyMessage.java @@ -504,7 +504,7 @@ public class RubyMessage extends RubyObject { break; case BYTES: case STRING: - Utils.validateStringEncoding(context.runtime, fieldDescriptor.getType(), value); + Utils.validateStringEncoding(context, fieldDescriptor.getType(), value); RubyString str = (RubyString) value; switch (fieldDescriptor.getType()) { case BYTES: @@ -695,7 +695,7 @@ public class RubyMessage extends RubyObject { } } if (addValue) { - Utils.checkType(context, fieldType, value, (RubyModule) typeClass); + value = Utils.checkType(context, fieldType, value, (RubyModule) typeClass); this.fields.put(fieldDescriptor, value); } else { this.fields.remove(fieldDescriptor); diff --git a/ruby/src/main/java/com/google/protobuf/jruby/RubyRepeatedField.java b/ruby/src/main/java/com/google/protobuf/jruby/RubyRepeatedField.java index 946f9e74..ae2907a9 100644 --- a/ruby/src/main/java/com/google/protobuf/jruby/RubyRepeatedField.java +++ b/ruby/src/main/java/com/google/protobuf/jruby/RubyRepeatedField.java @@ -110,7 +110,7 @@ public class RubyRepeatedField extends RubyObject { @JRubyMethod(name = "[]=") public IRubyObject indexSet(ThreadContext context, IRubyObject index, IRubyObject value) { int arrIndex = normalizeArrayIndex(index); - Utils.checkType(context, fieldType, value, (RubyModule) typeClass); + value = Utils.checkType(context, fieldType, value, (RubyModule) typeClass); IRubyObject defaultValue = defaultValue(context); for (int i = this.storage.size(); i < arrIndex; i++) { this.storage.set(i, defaultValue); @@ -166,7 +166,7 @@ public class RubyRepeatedField extends RubyObject { public IRubyObject push(ThreadContext context, IRubyObject value) { if (!(fieldType == Descriptors.FieldDescriptor.Type.MESSAGE && value == context.runtime.getNil())) { - Utils.checkType(context, fieldType, value, (RubyModule) typeClass); + value = Utils.checkType(context, fieldType, value, (RubyModule) typeClass); } this.storage.add(value); return this.storage; diff --git a/ruby/src/main/java/com/google/protobuf/jruby/Utils.java b/ruby/src/main/java/com/google/protobuf/jruby/Utils.java index 596a0979..f199feb9 100644 --- a/ruby/src/main/java/com/google/protobuf/jruby/Utils.java +++ b/ruby/src/main/java/com/google/protobuf/jruby/Utils.java @@ -64,8 +64,8 @@ public class Utils { return context.runtime.newSymbol(typeName.replace("TYPE_", "").toLowerCase()); } - public static void checkType(ThreadContext context, Descriptors.FieldDescriptor.Type fieldType, - IRubyObject value, RubyModule typeClass) { + public static IRubyObject checkType(ThreadContext context, Descriptors.FieldDescriptor.Type fieldType, + IRubyObject value, RubyModule typeClass) { Ruby runtime = context.runtime; Object val; switch(fieldType) { @@ -106,7 +106,7 @@ public class Utils { break; case BYTES: case STRING: - validateStringEncoding(context.runtime, fieldType, value); + value = validateStringEncoding(context, fieldType, value); break; case MESSAGE: if (value.getMetaClass() != typeClass) { @@ -127,6 +127,7 @@ public class Utils { default: break; } + return value; } public static IRubyObject wrapPrimaryValue(ThreadContext context, Descriptors.FieldDescriptor.Type fieldType, Object value) { @@ -148,10 +149,16 @@ public class Utils { return runtime.newFloat((Double) value); case BOOL: return (Boolean) value ? runtime.getTrue() : runtime.getFalse(); - case BYTES: - return runtime.newString(((ByteString) value).toStringUtf8()); - case STRING: - return runtime.newString(value.toString()); + case BYTES: { + IRubyObject wrapped = runtime.newString(((ByteString) value).toStringUtf8()); + wrapped.setFrozen(true); + return wrapped; + } + case STRING: { + IRubyObject wrapped = runtime.newString(value.toString()); + wrapped.setFrozen(true); + return wrapped; + } default: return runtime.getNil(); } @@ -180,25 +187,21 @@ public class Utils { } } - public static void validateStringEncoding(Ruby runtime, Descriptors.FieldDescriptor.Type type, IRubyObject value) { + public static IRubyObject validateStringEncoding(ThreadContext context, Descriptors.FieldDescriptor.Type type, IRubyObject value) { if (!(value instanceof RubyString)) - throw runtime.newTypeError("Invalid argument for string field."); - Encoding encoding = ((RubyString) value).getEncoding(); + throw context.runtime.newTypeError("Invalid argument for string field."); switch(type) { case BYTES: - if (encoding != ASCIIEncoding.INSTANCE) - throw runtime.newTypeError("Encoding for bytes fields" + - " must be \"ASCII-8BIT\", but was " + encoding); + value = ((RubyString)value).encode(context, context.runtime.evalScriptlet("Encoding::ASCII_8BIT")); break; case STRING: - if (encoding != UTF8Encoding.INSTANCE - && encoding != USASCIIEncoding.INSTANCE) - throw runtime.newTypeError("Encoding for string fields" + - " must be \"UTF-8\" or \"ASCII\", but was " + encoding); + value = ((RubyString)value).encode(context, context.runtime.evalScriptlet("Encoding::UTF_8")); break; default: break; } + value.setFrozen(true); + return value; } public static void checkNameAvailability(ThreadContext context, String name) { diff --git a/ruby/tests/basic.rb b/ruby/tests/basic.rb index fee07e33..8b6d329e 100644 --- a/ruby/tests/basic.rb +++ b/ruby/tests/basic.rb @@ -255,14 +255,17 @@ module BasicTest m = TestMessage.new # Assigning a normal (ASCII or UTF8) string to a bytes field, or - # ASCII-8BIT to a string field, raises an error. - assert_raise TypeError do - m.optional_bytes = "Test string ASCII".encode!('ASCII') - end - assert_raise TypeError do + # ASCII-8BIT to a string field will convert to the proper encoding. + m.optional_bytes = "Test string ASCII".encode!('ASCII') + assert m.optional_bytes.frozen? + assert_equal Encoding::ASCII_8BIT, m.optional_bytes.encoding + assert_equal "Test string ASCII", m.optional_bytes + + assert_raise Encoding::UndefinedConversionError do m.optional_bytes = "Test string UTF-8 \u0100".encode!('UTF-8') end - assert_raise TypeError do + + assert_raise Encoding::UndefinedConversionError do m.optional_string = ["FFFF"].pack('H*') end @@ -270,11 +273,10 @@ module BasicTest m.optional_bytes = ["FFFF"].pack('H*') m.optional_string = "\u0100" - # strings are mutable so we can do this, but serialize should catch it. + # strings are immutable so we can't do this, but serialize should catch it. m.optional_string = "asdf".encode!('UTF-8') - m.optional_string.encode!('ASCII-8BIT') - assert_raise TypeError do - data = TestMessage.encode(m) + assert_raise RuntimeError do + m.optional_string.encode!('ASCII-8BIT') end end @@ -466,9 +468,9 @@ module BasicTest assert m.length == 2 m2 = m.dup - assert m == m2 + assert_equal m, m2 assert m.hash != 0 - assert m.hash == m2.hash + assert_equal m.hash, m2.hash collected = {} m.each { |k,v| collected[v] = k } @@ -558,7 +560,7 @@ module BasicTest assert_raise TypeError do m[1] = 1 end - assert_raise TypeError do + assert_raise Encoding::UndefinedConversionError do bytestring = ["FFFF"].pack("H*") m[bytestring] = 1 end @@ -566,9 +568,8 @@ module BasicTest m = Google::Protobuf::Map.new(:bytes, :int32) bytestring = ["FFFF"].pack("H*") m[bytestring] = 1 - assert_raise TypeError do - m["asdf"] = 1 - end + # Allowed -- we will automatically convert to ASCII-8BIT. + m["asdf"] = 1 assert_raise TypeError do m[1] = 1 end @@ -853,15 +854,22 @@ module BasicTest def test_encode_decode_helpers m = TestMessage.new(:optional_string => 'foo', :repeated_string => ['bar1', 'bar2']) + assert_equal 'foo', m.optional_string + assert_equal ['bar1', 'bar2'], m.repeated_string + json = m.to_json m2 = TestMessage.decode_json(json) - assert m2.optional_string == 'foo' - assert m2.repeated_string == ['bar1', 'bar2'] + assert_equal 'foo', m2.optional_string + assert_equal ['bar1', 'bar2'], m2.repeated_string + if RUBY_PLATFORM != "java" + assert m2.optional_string.frozen? + assert m2.repeated_string[0].frozen? + end proto = m.to_proto m2 = TestMessage.decode(proto) - assert m2.optional_string == 'foo' - assert m2.repeated_string == ['bar1', 'bar2'] + assert_equal 'foo', m2.optional_string + assert_equal ['bar1', 'bar2'], m2.repeated_string end def test_protobuf_encode_decode_helpers diff --git a/ruby/tests/generated_code.proto b/ruby/tests/generated_code.proto index 42d82a6b..62fd83ed 100644 --- a/ruby/tests/generated_code.proto +++ b/ruby/tests/generated_code.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package A.B.C; +package a.b.c; message TestMessage { int32 optional_int32 = 1; diff --git a/ruby/tests/generated_code_test.rb b/ruby/tests/generated_code_test.rb index daef357a..b92b0462 100644 --- a/ruby/tests/generated_code_test.rb +++ b/ruby/tests/generated_code_test.rb @@ -3,7 +3,8 @@ # generated_code.rb is in the same directory as this test. $LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__))) -require 'generated_code' +require 'generated_code_pb' +require 'test_import_pb' require 'test/unit' class GeneratedCodeTest < Test::Unit::TestCase @@ -13,5 +14,6 @@ class GeneratedCodeTest < Test::Unit::TestCase # successfully creates message definitions and classes, not to test every # aspect of the extension (basic.rb is for that). m = A::B::C::TestMessage.new() + m2 = FooBar::TestImportedMessage.new() end end diff --git a/ruby/tests/test_import.proto b/ruby/tests/test_import.proto new file mode 100644 index 00000000..230484ee --- /dev/null +++ b/ruby/tests/test_import.proto @@ -0,0 +1,5 @@ +syntax = "proto3"; + +package foo_bar; + +message TestImportedMessage {} diff --git a/ruby/tests/well_known_types_test.rb b/ruby/tests/well_known_types_test.rb new file mode 100644 index 00000000..9b46632b --- /dev/null +++ b/ruby/tests/well_known_types_test.rb @@ -0,0 +1,122 @@ +#!/usr/bin/ruby + +require 'test/unit' +require 'google/protobuf/well_known_types' + +class TestWellKnownTypes < Test::Unit::TestCase + def test_timestamp + ts = Google::Protobuf::Timestamp.new + + assert_equal Time.at(0), ts.to_time + + ts.seconds = 12345 + assert_equal Time.at(12345), ts.to_time + assert_equal 12345, ts.to_i + + ts.from_time(Time.at(123456, 654321)) + assert_equal 123456, ts.seconds + assert_equal 654321000, ts.nanos + assert_equal Time.at(123456.654321), ts.to_time + end + + def test_duration + duration = Google::Protobuf::Duration.new(seconds: 123, nanos: 456) + assert_equal 123.000000456, duration.to_f + end + + def test_struct + struct = Google::Protobuf::Struct.new + + substruct = { + "subkey" => 999, + "subkey2" => false + } + + sublist = ["abc", 123, {"deepkey" => "deepval"}] + + struct["number"] = 12345 + struct["boolean-true"] = true + struct["boolean-false"] = false + struct["null"] = nil + struct["string"] = "abcdef" + struct["substruct"] = substruct + struct["sublist"] = sublist + + assert_equal 12345, struct["number"] + assert_equal true, struct["boolean-true"] + assert_equal false, struct["boolean-false"] + assert_equal nil, struct["null"] + assert_equal "abcdef", struct["string"] + assert_equal(Google::Protobuf::Struct.from_hash(substruct), + struct["substruct"]) + assert_equal(Google::Protobuf::ListValue.from_a(sublist), + struct["sublist"]) + + should_equal = { + "number" => 12345, + "boolean-true" => true, + "boolean-false" => false, + "null" => nil, + "string" => "abcdef", + "substruct" => { + "subkey" => 999, + "subkey2" => false + }, + "sublist" => ["abc", 123, {"deepkey" => "deepval"}] + } + + list = struct["sublist"] + list.is_a?(Google::Protobuf::ListValue) + assert_equal "abc", list[0] + assert_equal 123, list[1] + assert_equal({"deepkey" => "deepval"}, list[2].to_h) + + # to_h returns a fully-flattened Ruby structure (Hash and Array). + assert_equal(should_equal, struct.to_h) + + # Test that we can assign Struct and ListValue directly. + struct["substruct"] = Google::Protobuf::Struct.from_hash(substruct) + struct["sublist"] = Google::Protobuf::ListValue.from_a(sublist) + + assert_equal(should_equal, struct.to_h) + + struct["sublist"] << nil + should_equal["sublist"] << nil + + assert_equal(should_equal, struct.to_h) + assert_equal(should_equal["sublist"].length, struct["sublist"].length) + + assert_raise Google::Protobuf::UnexpectedStructType do + struct[123] = 5 + end + + assert_raise Google::Protobuf::UnexpectedStructType do + struct[5] = Time.new + end + + assert_raise Google::Protobuf::UnexpectedStructType do + struct[5] = [Time.new] + end + + assert_raise Google::Protobuf::UnexpectedStructType do + struct[5] = {123 => 456} + end + + assert_raise Google::Protobuf::UnexpectedStructType do + struct = Google::Protobuf::Struct.new + struct.fields["foo"] = Google::Protobuf::Value.new + # Tries to return a Ruby value for a Value class whose type + # hasn't been filled in. + struct["foo"] + end + end + + def test_any + any = Google::Protobuf::Any.new + ts = Google::Protobuf::Timestamp.new(seconds: 12345, nanos: 6789) + any.pack(ts) + + assert any.is(Google::Protobuf::Timestamp) + assert_equal ts, any.unpack(Google::Protobuf::Timestamp) + end +end diff --git a/ruby/travis-test.sh b/ruby/travis-test.sh index 75db7d93..59e970a6 100755 --- a/ruby/travis-test.sh +++ b/ruby/travis-test.sh @@ -5,7 +5,7 @@ set -e test_version() { version=$1 - if [ "$version" == "jruby" ] ; then + if [ "$version" == "jruby-1.7" ] ; then # No conformance tests yet -- JRuby is too broken to run them. bash --login -c \ "rvm install $version && rvm use $version && \ |