diff options
author | Joshua Haberman <jhaberman@gmail.com> | 2016-02-19 14:39:17 -0800 |
---|---|---|
committer | Joshua Haberman <jhaberman@gmail.com> | 2016-02-19 14:39:17 -0800 |
commit | 32daf513ced8d51e8de6cc8d800cfc972c4df5d6 (patch) | |
tree | d430af9b032a81e73757c59209a1ed1d6e9be42e /js | |
parent | 894c4d6a8bb9ab6b6d4791be98656cfb5399d0bc (diff) | |
parent | 24c5424be5b220d8d4575e4262ee5d6ad9417959 (diff) | |
download | protobuf-32daf513ced8d51e8de6cc8d800cfc972c4df5d6.tar.gz protobuf-32daf513ced8d51e8de6cc8d800cfc972c4df5d6.tar.bz2 protobuf-32daf513ced8d51e8de6cc8d800cfc972c4df5d6.zip |
Merge pull request #1215 from haberman/commonjs
Added support for CommonJS require()
Diffstat (limited to 'js')
-rw-r--r-- | js/README.md | 165 | ||||
-rw-r--r-- | js/binary/proto_test.js | 2 | ||||
-rw-r--r-- | js/commonjs/export.js | 22 | ||||
-rw-r--r-- | js/commonjs/export_asserts.js | 37 | ||||
-rw-r--r-- | js/commonjs/jasmine.json | 9 | ||||
-rw-r--r-- | js/commonjs/rewrite_tests_for_commonjs.js | 92 | ||||
-rw-r--r-- | js/debug_test.js | 5 | ||||
-rw-r--r-- | js/gulpfile.js | 79 | ||||
-rw-r--r-- | js/message_test.js | 24 | ||||
-rw-r--r-- | js/package.json | 7 | ||||
-rw-r--r-- | js/proto3_test.js | 4 | ||||
-rw-r--r-- | js/test.proto | 7 |
12 files changed, 431 insertions, 22 deletions
diff --git a/js/README.md b/js/README.md index fc144a3d..15d48c87 100644 --- a/js/README.md +++ b/js/README.md @@ -1,14 +1,159 @@ -This directory contains Protocol Buffer support for JavaScript. This code works -in browsers and in Node.js. +Protocol Buffers - Google's data interchange format +=================================================== -The packaging work for this is still in-progress. For now you can just run the -tests. First you need to build the main C++ distribution because the code -generator for JavaScript is written in C++: +[![Build Status](https://travis-ci.org/google/protobuf.svg?branch=master)](https://travis-ci.org/google/protobuf) - $ ./autogen.sh - $ ./configure - $ make +Copyright 2008 Google Inc. -Then you can run the JavaScript tests in this directory: +This directory contains the JavaScript Protocol Buffers runtime library. - $ cd js && gulp test +The library is currently compatible with: + +1. CommonJS-style imports (eg. `var protos = require('my-protos');`) +2. Closure-style imports (eg. `goog.require('my.package.MyProto');`) + +Support for ES6-style imports is not implemented yet. Browsers can +be supported by using Browserify, webpack, Closure Compiler, etc. to +resolve imports at compile time. + +To use Protocol Buffers with JavaScript, you need two main components: + +1. The protobuf runtime library. You can install this with + `npm install google-protobuf`, or use the files in this directory. +2. The Protocol Compiler `protoc`. This translates `.proto` files + into `.js` files. The compiler is not currently available via + npm, but you can download a pre-built binary + [on GitHub](https://github.com/google/protobuf/releases) + (look for the `protoc-*.zip` files under **Downloads**). + + +Setup +===== + +First, obtain the Protocol Compiler. The easiest way is to download +a pre-built binary from [https://github.com/google/protobuf/releases](https://github.com/google/protobuf/releases). + +If you want, you can compile `protoc` from source instead. To do this +follow the instructions in [the top-level +README](https://github.com/google/protobuf/blob/master/src/README.md). + +Once you have `protoc` compiled, you can run the tests by typing: + + $ cd js + $ npm install + $ npm test + + # If your protoc is somewhere else than ../src/protoc, instead do this. + # But make sure your protoc is the same version as this (or compatible)! + $ PROTOC=/usr/local/bin/protoc npm test + +This will run two separate copies of the tests: one that uses +Closure Compiler style imports and one that uses CommonJS imports. +You can see all the CommonJS files in `commonjs_out/`. +If all of these tests pass, you know you have a working setup. + + +Using Protocol Buffers in your own project +========================================== + +To use Protocol Buffers in your own project, you need to integrate +the Protocol Compiler into your build system. The details are a +little different depending on whether you are using Closure imports +or CommonJS imports: + +Closure Imports +--------------- + +If you want to use Closure imports, your build should run a command +like this: + + $ protoc --js_out=library=myproto_libs,binary:. messages.proto base.proto + +For Closure imports, `protoc` will generate a single output file +(`myproto_libs.js` in this example). The generated file will `goog.provide()` +all of the types defined in your .proto files. For example, for the unit +tests the generated files contain many `goog.provide` statements like: + + goog.provide('proto.google.protobuf.DescriptorProto'); + goog.provide('proto.google.protobuf.DescriptorProto.ExtensionRange'); + goog.provide('proto.google.protobuf.DescriptorProto.ReservedRange'); + goog.provide('proto.google.protobuf.EnumDescriptorProto'); + goog.provide('proto.google.protobuf.EnumOptions'); + +The generated code will also `goog.require()` many types in the core library, +and they will require many types in the Google Closure library. So make sure +that your `goog.provide()` / `goog.require()` setup can find all of your +generated code, the core library `.js` files in this directory, and the +Google Closure library itself. + +Once you've done this, you should be able to import your types with +statements like: + + goog.require('proto.my.package.MyMessage'); + + var message = proto.my.package.MyMessage(); + +CommonJS imports +---------------- + +If you want to use CommonJS imports, your build should run a command +like this: + + $ protoc --js_out=import_style=commonjs,binary:. messages.proto base.proto + +For CommonJS imports, `protoc` will spit out one file per input file +(so `messages_pb.js` and `base_pb.js` in this example). The generated +code will depend on the core runtime, which should be in a file called +`google-protobuf.js`. If you are installing from `npm`, this file should +already be built and available. If you are running from GitHub, you need +to build it first by running: + + $ gulp dist + +Once you've done this, you should be able to import your types with +statements like: + + var messages = require('./messages_pb'); + + var message = new messages.MyMessage(); + +The `--js_out` flag +------------------- + +The syntax of the `--js_out` flag is: + + --js_out=[OPTIONS:]output_dir + +Where `OPTIONS` are separated by commas. Options are either `opt=val` or +just `opt` (for options that don't take a value). The available options +are specified and documented in the `GeneratorOptions` struct in +[src/google/protobuf/compiler/js/js_generator.h](https://github.com/google/protobuf/blob/master/src/google/protobuf/compiler/js/js_generator.h#L53). + +Some examples: + +- `--js_out=library=myprotos_lib.js,binary:.`: this contains the options + `library=myprotos.lib.js` and `binary` and outputs to the current directory. + The `import_style` option is left to the default, which is `closure`. +- `--js_out=import_style=commonjs,binary:protos`: this contains the options + `import_style=commonjs` and `binary` and outputs to the directory `protos`. + +API +=== + +The API is not well-documented yet. Here is a quick example to give you an +idea of how the library generally works: + + var message = new MyMessage(); + + message.setName("John Doe"); + message.setAge(25); + message.setPhoneNumbers(["800-555-1212", "800-555-0000"]); + + // Serializes to a UInt8Array. + bytes = message.serializeBinary(); + + var message2 = new MyMessage(); + message2.deserializeBinary(bytes); + +For more examples, see the tests. You can also look at the generated code +to see what methods are defined for your generated messages. diff --git a/js/binary/proto_test.js b/js/binary/proto_test.js index 1cb7ff0e..817f8a79 100644 --- a/js/binary/proto_test.js +++ b/js/binary/proto_test.js @@ -31,6 +31,8 @@ // Test suite is written using Jasmine -- see http://jasmine.github.io/ goog.require('goog.testing.asserts'); + +// CommonJS-LoadFromFile: testbinary_pb proto.jspb.test goog.require('proto.jspb.test.ExtendsWithMessage'); goog.require('proto.jspb.test.ForeignEnum'); goog.require('proto.jspb.test.ForeignMessage'); diff --git a/js/commonjs/export.js b/js/commonjs/export.js new file mode 100644 index 00000000..a3cfbd6f --- /dev/null +++ b/js/commonjs/export.js @@ -0,0 +1,22 @@ +/** + * @fileoverview Export symbols needed by generated code in CommonJS style. + * + * This effectively is our canonical list of what we publicly export from + * the google-protobuf.js file that we build at distribution time. + */ + +goog.require('goog.object'); +goog.require('jspb.BinaryReader'); +goog.require('jspb.BinaryWriter'); +goog.require('jspb.ExtensionFieldInfo'); +goog.require('jspb.Message'); + +exports.Message = jspb.Message; +exports.BinaryReader = jspb.BinaryReader; +exports.BinaryWriter = jspb.BinaryWriter; +exports.ExtensionFieldInfo = jspb.ExtensionFieldInfo; + +// These are used by generated code but should not be used directly by clients. +exports.exportSymbol = goog.exportSymbol; +exports.inherits = goog.inherits; +exports.object = {extend: goog.object.extend}; diff --git a/js/commonjs/export_asserts.js b/js/commonjs/export_asserts.js new file mode 100644 index 00000000..5219d120 --- /dev/null +++ b/js/commonjs/export_asserts.js @@ -0,0 +1,37 @@ +/** + * @fileoverview Exports symbols needed only by tests. + * + * This file exports several Closure Library symbols that are only + * used by tests. It is used to generate a file + * closure_asserts_commonjs.js that is only used at testing time. + */ + +goog.require('goog.testing.asserts'); + +var global = Function('return this')(); + +// All of the closure "assert" functions are exported at the global level. +// +// The Google Closure assert functions start with assert, eg. +// assertThrows +// assertNotThrows +// assertTrue +// ... +// +// The one exception is the "fail" function. +function shouldExport(str) { + return str.lastIndexOf('assert') === 0 || str == 'fail'; +} + +for (var key in global) { + if ((typeof key == "string") && global.hasOwnProperty(key) && + shouldExport(key)) { + exports[key] = global[key]; + } +} + +// The COMPILED variable is set by Closure compiler to "true" when it compiles +// JavaScript, so in practice this is equivalent to "exports.COMPILED = true". +// This will disable some debugging functionality in debug.js. We could +// investigate whether this can/should be enabled in CommonJS builds. +exports.COMPILED = COMPILED diff --git a/js/commonjs/jasmine.json b/js/commonjs/jasmine.json new file mode 100644 index 00000000..666b8edb --- /dev/null +++ b/js/commonjs/jasmine.json @@ -0,0 +1,9 @@ +{ + "spec_dir": "", + "spec_files": [ + "*_test.js", + "binary/proto_test.js" + ], + "helpers": [ + ] +} diff --git a/js/commonjs/rewrite_tests_for_commonjs.js b/js/commonjs/rewrite_tests_for_commonjs.js new file mode 100644 index 00000000..dc5effec --- /dev/null +++ b/js/commonjs/rewrite_tests_for_commonjs.js @@ -0,0 +1,92 @@ +/** + * @fileoverview Utility to translate test files to CommonJS imports. + * + * This is a somewhat hacky tool designed to do one very specific thing. + * All of the test files in *_test.js are written with Closure-style + * imports (goog.require()). This works great for running the tests + * against Closure-style generated code, but we also want to run the + * tests against CommonJS-style generated code without having to fork + * the tests. + * + * Closure-style imports import each individual type by name. This is + * very different than CommonJS imports which are by file. So we put + * special comments in these tests like: + * + * // CommonJS-LoadFromFile: test_pb + * goog.require('proto.jspb.test.CloneExtension'); + * goog.require('proto.jspb.test.Complex'); + * goog.require('proto.jspb.test.DefaultValues'); + * + * This script parses that special comment and uses it to generate proper + * CommonJS require() statements so that the tests can run and pass using + * CommonJS imports. The script will change the above statements into: + * + * var test_pb = require('test_pb'); + * googleProtobuf.exportSymbol('proto.jspb.test.CloneExtension', test_pb.CloneExtension, global); + * googleProtobuf.exportSymbol('proto.jspb.test.Complex', test_pb.Complex, global); + * googleProtobuf.exportSymbol('proto.jspb.test.DefaultValues', test_pb.DefaultValues, global); + * + * (The "exportSymbol" function will define the given names in the global + * namespace, taking care not to overwrite any previous value for + * "proto.jspb.test"). + */ + +var lineReader = require('readline').createInterface({ + input: process.stdin, + output: process.stdout +}); + +function tryStripPrefix(str, prefix) { + if (str.lastIndexOf(prefix) !== 0) { + throw "String: " + str + " didn't start with: " + prefix; + } + return str.substr(prefix.length); +} + +function camelCase(str) { + var ret = ''; + var ucaseNext = false; + for (var i = 0; i < str.length; i++) { + if (str[i] == '-') { + ucaseNext = true; + } else if (ucaseNext) { + ret += str[i].toUpperCase(); + ucaseNext = false; + } else { + ret += str[i]; + } + } + return ret; +} + +var module = null; +var pkg = null; +lineReader.on('line', function(line) { + var isRequire = line.match(/goog\.require\('([^']*)'\)/); + var isLoadFromFile = line.match(/CommonJS-LoadFromFile: (\S*) (.*)/); + var isSetTestOnly = line.match(/goog.setTestOnly()/); + if (isRequire) { + if (module) { // Skip goog.require() lines before the first directive. + var fullSym = isRequire[1]; + var sym = tryStripPrefix(fullSym, pkg); + console.log("googleProtobuf.exportSymbol('" + fullSym + "', " + module + sym + ', global);'); + } + } else if (isLoadFromFile) { + if (!module) { + console.log("var googleProtobuf = require('google-protobuf');"); + console.log("var asserts = require('closure_asserts_commonjs');"); + console.log("var global = Function('return this')();"); + console.log(""); + console.log("// Bring asserts into the global namespace."); + console.log("googleProtobuf.object.extend(global, asserts);"); + } + module = camelCase(isLoadFromFile[1]) + pkg = isLoadFromFile[2]; + + if (module != "googleProtobuf") { // We unconditionally require this in the header. + console.log("var " + module + " = require('" + isLoadFromFile[1] + "');"); + } + } else if (!isSetTestOnly) { // Remove goog.setTestOnly() lines. + console.log(line); + } +}); diff --git a/js/debug_test.js b/js/debug_test.js index 615fc7c6..d7bf3768 100644 --- a/js/debug_test.js +++ b/js/debug_test.js @@ -31,13 +31,16 @@ goog.setTestOnly(); goog.require('goog.testing.asserts'); + +// CommonJS-LoadFromFile: google-protobuf goog.require('jspb.debug'); + +// CommonJS-LoadFromFile: test_pb goog.require('proto.jspb.test.HasExtensions'); goog.require('proto.jspb.test.IsExtension'); goog.require('proto.jspb.test.Simple1'); - describe('debugTest', function() { it('testSimple1', function() { if (COMPILED) { diff --git a/js/gulpfile.js b/js/gulpfile.js index 79095d65..b0faed06 100644 --- a/js/gulpfile.js +++ b/js/gulpfile.js @@ -1,25 +1,79 @@ var gulp = require('gulp'); var exec = require('child_process').exec; +var glob = require('glob'); -gulp.task('genproto', function (cb) { - exec('../src/protoc --js_out=library=testproto_libs,binary:. -I ../src -I . *.proto ../src/google/protobuf/descriptor.proto', +var protoc = process.env.PROTOC || '../src/protoc'; + +gulp.task('genproto_closure', function (cb) { + exec(protoc + ' --js_out=library=testproto_libs,binary:. -I ../src -I . *.proto ../src/google/protobuf/descriptor.proto', + function (err, stdout, stderr) { + console.log(stdout); + console.log(stderr); + cb(err); + }); +}); + +gulp.task('genproto_commonjs', function (cb) { + exec('mkdir -p commonjs_out && ' + protoc + ' --js_out=import_style=commonjs,binary:commonjs_out -I ../src -I . *.proto ../src/google/protobuf/descriptor.proto', + function (err, stdout, stderr) { + console.log(stdout); + console.log(stderr); + cb(err); + }); +}); + +gulp.task('dist', function (cb) { + // TODO(haberman): minify this more aggressively. + // Will require proper externs/exports. + exec('./node_modules/google-closure-library/closure/bin/calcdeps.py -i message.js -i binary/reader.js -i binary/writer.js -i commonjs/export.js -p . -p node_modules/google-closure-library/closure -o compiled --compiler_jar node_modules/google-closure-compiler/compiler.jar > google-protobuf.js', + function (err, stdout, stderr) { + console.log(stdout); + console.log(stderr); + cb(err); + }); +}); + +gulp.task('commonjs_asserts', function (cb) { + exec('mkdir -p commonjs_out && ./node_modules/google-closure-library/closure/bin/calcdeps.py -i commonjs/export_asserts.js -p . -p node_modules/google-closure-library/closure -o compiled --compiler_jar node_modules/google-closure-compiler/compiler.jar > commonjs_out/closure_asserts_commonjs.js', function (err, stdout, stderr) { console.log(stdout); console.log(stderr); cb(err); }); -}) +}); + +gulp.task('make_commonjs_out', ['dist', 'genproto_commonjs', 'commonjs_asserts'], function (cb) { + // TODO(haberman): minify this more aggressively. + // Will require proper externs/exports. + var cmd = "mkdir -p commonjs_out/binary && "; + function addTestFile(file) { + cmd += 'node commonjs/rewrite_tests_for_commonjs.js < ' + file + + ' > commonjs_out/' + file + '&& '; + } + + glob.sync('*_test.js').forEach(addTestFile); + glob.sync('binary/*_test.js').forEach(addTestFile); + + exec(cmd + + 'cp commonjs/jasmine.json commonjs_out/jasmine.json && ' + + 'cp google-protobuf.js commonjs_out', + function (err, stdout, stderr) { + console.log(stdout); + console.log(stderr); + cb(err); + }); +}); -gulp.task('deps', ['genproto'], function (cb) { +gulp.task('deps', ['genproto_closure'], function (cb) { exec('./node_modules/google-closure-library/closure/bin/build/depswriter.py *.js binary/*.js > deps.js', function (err, stdout, stderr) { console.log(stdout); console.log(stderr); cb(err); }); -}) +}); -gulp.task('test', ['genproto', 'deps'], function (cb) { +gulp.task('test_closure', ['genproto_closure', 'deps'], function (cb) { exec('JASMINE_CONFIG_PATH=jasmine.json ./node_modules/.bin/jasmine', function (err, stdout, stderr) { console.log(stdout); @@ -27,3 +81,16 @@ gulp.task('test', ['genproto', 'deps'], function (cb) { cb(err); }); }); + +gulp.task('test_commonjs', ['make_commonjs_out'], function (cb) { + exec('cd commonjs_out && JASMINE_CONFIG_PATH=jasmine.json NODE_PATH=. ../node_modules/.bin/jasmine', + function (err, stdout, stderr) { + console.log(stdout); + console.log(stderr); + cb(err); + }); +}); + +gulp.task('test', ['test_closure', 'test_commonjs'], function(cb) { + cb(); +}); diff --git a/js/message_test.js b/js/message_test.js index 971ea4f4..f572188e 100644 --- a/js/message_test.js +++ b/js/message_test.js @@ -34,35 +34,47 @@ goog.setTestOnly(); goog.require('goog.json'); goog.require('goog.testing.asserts'); + +// CommonJS-LoadFromFile: google-protobuf jspb goog.require('jspb.Message'); + +// CommonJS-LoadFromFile: test5_pb proto.jspb.exttest.beta goog.require('proto.jspb.exttest.beta.floatingStrField'); + +// CommonJS-LoadFromFile: test3_pb proto.jspb.exttest goog.require('proto.jspb.exttest.floatingMsgField'); + +// CommonJS-LoadFromFile: test4_pb proto.jspb.exttest goog.require('proto.jspb.exttest.floatingMsgFieldTwo'); + +// CommonJS-LoadFromFile: test_pb proto.jspb.test goog.require('proto.jspb.test.CloneExtension'); goog.require('proto.jspb.test.Complex'); goog.require('proto.jspb.test.DefaultValues'); goog.require('proto.jspb.test.Empty'); goog.require('proto.jspb.test.EnumContainer'); -goog.require('proto.jspb.test.ExtensionMessage'); -goog.require('proto.jspb.test.floatingMsgField'); goog.require('proto.jspb.test.floatingStrField'); goog.require('proto.jspb.test.HasExtensions'); goog.require('proto.jspb.test.IndirectExtension'); goog.require('proto.jspb.test.IsExtension'); goog.require('proto.jspb.test.OptionalFields'); goog.require('proto.jspb.test.OuterEnum'); +goog.require('proto.jspb.test.OuterMessage.Complex'); goog.require('proto.jspb.test.simple1'); goog.require('proto.jspb.test.Simple1'); goog.require('proto.jspb.test.Simple2'); goog.require('proto.jspb.test.SpecialCases'); goog.require('proto.jspb.test.TestClone'); -goog.require('proto.jspb.test.TestExtensionsMessage'); goog.require('proto.jspb.test.TestGroup'); goog.require('proto.jspb.test.TestGroup1'); goog.require('proto.jspb.test.TestMessageWithOneof'); goog.require('proto.jspb.test.TestReservedNames'); goog.require('proto.jspb.test.TestReservedNamesExtension'); +// CommonJS-LoadFromFile: test2_pb proto.jspb.test +goog.require('proto.jspb.test.ExtensionMessage'); +goog.require('proto.jspb.test.TestExtensionsMessage'); +goog.require('proto.jspb.test.floatingMsgField'); @@ -86,6 +98,12 @@ describe('Message test suite', function() { assertEquals('some_bytes', data.getBytesField()); }); + it('testNestedMessage', function() { + var msg = new proto.jspb.test.OuterMessage.Complex(); + msg.setInnerComplexField(5); + assertObjectEquals({innerComplexField: 5}, msg.toObject()); + }); + it('testComplexConversion', function() { var data1 = ['a',,, [, 11], [[, 22], [, 33]],, ['s1', 's2'],, 1]; var data2 = ['a',,, [, 11], [[, 22], [, 33]],, ['s1', 's2'],, 1]; diff --git a/js/package.json b/js/package.json index be93286f..6418e507 100644 --- a/js/package.json +++ b/js/package.json @@ -2,13 +2,16 @@ "name": "google-protobuf", "version": "3.0.0-alpha.5", "description": "Protocol Buffers for JavaScript", - "main": "debug.js", + "main": "google-protobuf.js", "dependencies": { "google-closure-library": "~20160125.0.0", "gulp": "~3.9.0", "jasmine": "~2.4.1" }, - "devDependencies": {}, + "devDependencies": { + "google-closure-compiler": "~20151216.2.0", + "glob": "~6.0.4" + }, "scripts": { "test": "./node_modules/gulp/bin/gulp.js test" }, diff --git a/js/proto3_test.js b/js/proto3_test.js index 8102bab6..f8868716 100644 --- a/js/proto3_test.js +++ b/js/proto3_test.js @@ -29,7 +29,11 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. goog.require('goog.testing.asserts'); + +// CommonJS-LoadFromFile: testbinary_pb proto.jspb.test goog.require('proto.jspb.test.ForeignMessage'); + +// CommonJS-LoadFromFile: proto3_test_pb proto.jspb.test goog.require('proto.jspb.test.Proto3Enum'); goog.require('proto.jspb.test.TestProto3'); diff --git a/js/test.proto b/js/test.proto index 5f9078ef..3cea5f37 100644 --- a/js/test.proto +++ b/js/test.proto @@ -100,6 +100,13 @@ message Complex { repeated string a_repeated_string = 7; } +message OuterMessage { + // Make sure this doesn't conflict with the other Complex message. + message Complex { + optional int32 inner_complex_field = 1; + } +} + message IsExtension { extend HasExtensions { optional IsExtension ext_field = 100; |