diff --git a/CONTRIBUTORS b/CONTRIBUTORS index a9bb457c..47f27bd7 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -15,6 +15,7 @@ Alexander Torstling (atorstling) Andrey Batalev Brice Figureau (masterzen) +Chris Heisterkamp (cheister) Daniel Evans (devans) David Trott (dtrott) Gregory Kick diff --git a/src/it/TEST-25/pom.xml b/src/it/TEST-25/pom.xml index 260256dd..07a69499 100644 --- a/src/it/TEST-25/pom.xml +++ b/src/it/TEST-25/pom.xml @@ -33,6 +33,10 @@ Integration Test 25 + + 3.4.0 + + @@ -45,7 +49,7 @@ compile - com.google.protobuf:protoc:3.4.0:exe:${os.detected.classifier} + com.google.protobuf:protoc:${protobufVersion}:exe:${os.detected.classifier} diff --git a/src/it/TEST-25/src/main/proto/test.proto b/src/it/TEST-25/src/main/proto/test.proto index a7446825..1e406cb7 100644 --- a/src/it/TEST-25/src/main/proto/test.proto +++ b/src/it/TEST-25/src/main/proto/test.proto @@ -14,6 +14,8 @@ // limitations under the License. // +syntax = "proto3"; + option java_package = "test"; option java_outer_classname = "TestProtos"; option optimize_for = SPEED; diff --git a/src/it/TEST-29/invoker.properties b/src/it/TEST-29/invoker.properties new file mode 100644 index 00000000..7aaa452a --- /dev/null +++ b/src/it/TEST-29/invoker.properties @@ -0,0 +1,22 @@ +# +# Copyright (c) 2016 Maven Protocol Buffers Plugin Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# An optional description for this build job to be included in the build reports. +invoker.description = Verifies that protoc can be invoked with an @args file and that unicode file names are properly handled. + +# A comma or space separated list of goals/phases to execute, may +# specify an empty list to execute the default goal of the IT project +invoker.goals = clean generate-sources diff --git a/src/it/TEST-29/pom.xml b/src/it/TEST-29/pom.xml new file mode 100644 index 00000000..f7681cf9 --- /dev/null +++ b/src/it/TEST-29/pom.xml @@ -0,0 +1,62 @@ + + + + + + + 4.0.0 + + + org.xolstice.maven.plugins.protobuf.its + it-parent + 1.0.0 + + + test-29 + 1.0.0 + + Integration Test 29 + + + + 3.5.1 + + + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + @project.version@ + + + + compile + + + com.google.protobuf:protoc:${protobufVersion}:exe:${os.detected.classifier} + + true + + + + + + + diff --git a/src/it/TEST-29/src/main/proto/test.proto b/src/it/TEST-29/src/main/proto/test.proto new file mode 100644 index 00000000..1e406cb7 --- /dev/null +++ b/src/it/TEST-29/src/main/proto/test.proto @@ -0,0 +1,24 @@ +// +// Copyright (c) 2016 Maven Protocol Buffers Plugin Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +syntax = "proto3"; + +option java_package = "test"; +option java_outer_classname = "TestProtos"; +option optimize_for = SPEED; + +message TestMessage { +} diff --git "a/src/it/TEST-29/src/main/proto/\321\202\320\265\321\201\321\202.proto" "b/src/it/TEST-29/src/main/proto/\321\202\320\265\321\201\321\202.proto" new file mode 100644 index 00000000..8df26d71 --- /dev/null +++ "b/src/it/TEST-29/src/main/proto/\321\202\320\265\321\201\321\202.proto" @@ -0,0 +1,24 @@ +// +// Copyright (c) 2016 Maven Protocol Buffers Plugin Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +syntax = "proto3"; + +option java_package = "test"; +option java_outer_classname = "TestProtosRu"; +option optimize_for = SPEED; + +message TestMessageRu { +} diff --git "a/src/it/TEST-29/src/main/proto/\343\203\206\343\202\271\343\203\210.proto" "b/src/it/TEST-29/src/main/proto/\343\203\206\343\202\271\343\203\210.proto" new file mode 100644 index 00000000..7d8a6485 --- /dev/null +++ "b/src/it/TEST-29/src/main/proto/\343\203\206\343\202\271\343\203\210.proto" @@ -0,0 +1,24 @@ +// +// Copyright (c) 2016 Maven Protocol Buffers Plugin Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +syntax = "proto3"; + +option java_package = "test"; +option java_outer_classname = "TestProtosJp"; +option optimize_for = SPEED; + +message TestMessageJp { +} diff --git a/src/it/TEST-29/verify.groovy b/src/it/TEST-29/verify.groovy new file mode 100644 index 00000000..4d018856 --- /dev/null +++ b/src/it/TEST-29/verify.groovy @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2016 Maven Protocol Buffers Plugin Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +outputDirectory = new File(basedir, 'target/generated-sources/protobuf/java'); +assert outputDirectory.exists(); +assert outputDirectory.isDirectory(); + +generatedJavaFile = new File(outputDirectory, 'test/TestProtos.java'); +assert generatedJavaFile.exists(); +assert generatedJavaFile.isFile(); + +content = generatedJavaFile.text; +assert content.contains('package test'); +assert content.contains('class TestProtos'); +assert content.contains('class TestMessage'); + +generatedJavaFile = new File(outputDirectory, 'test/TestProtosJp.java'); +assert generatedJavaFile.exists(); +assert generatedJavaFile.isFile(); + +content = generatedJavaFile.text; +assert content.contains('package test'); +assert content.contains('class TestProtosJp'); +assert content.contains('class TestMessageJp'); + +generatedJavaFile = new File(outputDirectory, 'test/TestProtosRu.java'); +assert generatedJavaFile.exists(); +assert generatedJavaFile.isFile(); + +content = generatedJavaFile.text; +assert content.contains('package test'); +assert content.contains('class TestProtosRu'); +assert content.contains('class TestMessageRu'); + +return true; diff --git a/src/main/java/org/xolstice/maven/plugin/protobuf/AbstractProtocMojo.java b/src/main/java/org/xolstice/maven/plugin/protobuf/AbstractProtocMojo.java index 4b259e93..f87b01fa 100644 --- a/src/main/java/org/xolstice/maven/plugin/protobuf/AbstractProtocMojo.java +++ b/src/main/java/org/xolstice/maven/plugin/protobuf/AbstractProtocMojo.java @@ -173,6 +173,18 @@ abstract class AbstractProtocMojo extends AbstractMojo { ) private List remoteRepositories; + /** + * A directory where temporary files will be generated. + * + * @since 0.6.0 + */ + @Parameter( + required = true, + readonly = true, + defaultValue = "${project.build.directory}" + ) + private File tempDirectory; + /** * A directory where native launchers for java protoc plugins will be generated. * @@ -345,6 +357,19 @@ abstract class AbstractProtocMojo extends AbstractMojo { ) protected boolean includeSourceInfoInDescriptorSet; + /** + * If set to {@code true}, the arguments to protoc will be put in a file and run as an argument + * file. This is helpful if you are getting Command line is too long errors + * This is only supported for protoc 3.5.0 and higher + * + * @since 0.6.0 + */ + @Parameter( + required = false, + defaultValue = "false" + ) + protected boolean useArgumentFile; + /** * Specifies one of more custom protoc plugins, written in Java * and available as Maven artifacts. An executable plugin will be created @@ -661,6 +686,8 @@ protected void addProtocBuilderParameters(final Protoc.Builder protocBuilder) th includeDependenciesInDescriptorSet, includeSourceInfoInDescriptorSet); } + protocBuilder.setTempDirectory(tempDirectory); + protocBuilder.useArgumentFile(useArgumentFile); } /** diff --git a/src/main/java/org/xolstice/maven/plugin/protobuf/Protoc.java b/src/main/java/org/xolstice/maven/plugin/protobuf/Protoc.java index 9264fee4..ead2d9e9 100644 --- a/src/main/java/org/xolstice/maven/plugin/protobuf/Protoc.java +++ b/src/main/java/org/xolstice/maven/plugin/protobuf/Protoc.java @@ -26,6 +26,9 @@ import org.codehaus.plexus.util.cli.Commandline; import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.charset.Charset; import java.util.LinkedHashSet; import java.util.List; import java.util.Set; @@ -112,6 +115,16 @@ final class Protoc { */ private final StringStreamConsumer error; + /** + * A directory where temporary files will be generated. + */ + private final File tempDirectory; + + /** + * A boolean indicating if the parameters to protoc should be passed in an argument file. + */ + private final boolean useArgumentFile; + /** * Constructs a new instance. This should only be used by the {@link Builder}. * @@ -133,6 +146,8 @@ final class Protoc { * @param nativePluginId a unique id of a native plugin. * @param nativePluginExecutable path to the native plugin executable. * @param nativePluginParameter an optional parameter for a native plugin. + * @param tempDirectory a directory where temporary files will be generated. + * @param useArgumentFile If {@code true}, parameters to protoc will be put in an argument file */ private Protoc( final String executable, @@ -150,7 +165,9 @@ private Protoc( final File pluginDirectory, final String nativePluginId, final String nativePluginExecutable, - final String nativePluginParameter) { + final String nativePluginParameter, + final File tempDirectory, + final boolean useArgumentFile) { this.executable = checkNotNull(executable, "executable"); this.protoPathElements = checkNotNull(protoPath, "protoPath"); this.protoFiles = checkNotNull(protoFiles, "protoFiles"); @@ -167,6 +184,8 @@ private Protoc( this.nativePluginId = nativePluginId; this.nativePluginExecutable = nativePluginExecutable; this.nativePluginParameter = nativePluginParameter; + this.tempDirectory = tempDirectory; + this.useArgumentFile = useArgumentFile; this.error = new StringStreamConsumer(); this.output = new StringStreamConsumer(); } @@ -174,13 +193,26 @@ private Protoc( /** * Invokes the {@code protoc} compiler using the configuration specified at construction. * + * @param log logger instance. * @return The exit status of {@code protoc}. * @throws CommandLineException if command line environment cannot be set up. + * @throws InterruptedException if the execution was interrupted by the user. */ public int execute(final Log log) throws CommandLineException, InterruptedException { final Commandline cl = new Commandline(); cl.setExecutable(executable); - cl.addArguments(buildProtocCommand().toArray(new String[] {})); + String[] args = buildProtocCommand().toArray(new String[] {}); + if (useArgumentFile) { + try { + File argumentsFile = createFileWithArguments(args); + log.debug(LOG_PREFIX + "Using arguments file " + argumentsFile.getPath()); + cl.addArguments(new String[] {"@" + argumentsFile.getAbsolutePath()}); + } catch (IOException e) { + log.error(LOG_PREFIX + "Error creating file with protoc arguments", e); + } + } else { + cl.addArguments(args); + } // There is a race condition in JDK that may sporadically prevent process creation on Linux // https://bugs.openjdk.java.net/browse/JDK-8068370 // In order to mitigate that, retry up to 2 more times before giving up @@ -334,14 +366,52 @@ public void logExecutionParameters(final Log log) { * @return the output */ public String getOutput() { - return output.getOutput(); + return fixUnicodeOutput(output.getOutput()); } /** * @return the error */ public String getError() { - return error.getOutput(); + return fixUnicodeOutput(error.getOutput()); + } + + /** + * Transcodes the output from system default charset to UTF-8. + * Protoc emits messages in UTF-8, but they are captured into a stream that has a system-default encoding. + * + * @param message a UTF-8 message in system-default encoding. + * @return the same message converted into a unicode string. + */ + private static String fixUnicodeOutput(final String message) { + return new String(message.getBytes(), Charset.forName("UTF-8")); + } + + /** + * Put args into a temp file to be referenced using the @ option in protoc command line. + * + * @param args + * @return the temporary file wth the arguments + * @throws IOException + */ + private File createFileWithArguments(String[] args) throws IOException { + PrintWriter writer = null; + try { + final File tempFile = File.createTempFile("protoc", null, tempDirectory); + tempFile.deleteOnExit(); + + writer = new PrintWriter(tempFile, "UTF-8"); + for (final String arg : args) { + writer.println(arg); + } + writer.flush(); + + return tempFile; + } finally { + if (writer != null) { + writer.close(); + } + } } /** @@ -360,6 +430,8 @@ static final class Builder { private final Set plugins; + private File tempDirectory; + private File pluginDirectory; // TODO reorganise support for custom plugins @@ -403,6 +475,8 @@ static final class Builder { private boolean includeSourceInfoInDescriptorSet; + private boolean useArgumentFile; + /** * Constructs a new builder. * @@ -416,6 +490,13 @@ static final class Builder { this.plugins = new LinkedHashSet(); } + public Builder setTempDirectory(final File directory) { + checkNotNull(directory); + checkArgument(directory.isDirectory(), "Temp directory " + directory + "does not exist"); + tempDirectory = directory; + return this; + } + /** * Sets the directory into which Java source files will be generated. * @@ -534,7 +615,7 @@ public Builder setPluginDirectory(final File directory) { return this; } - public void setNativePluginId(final String nativePluginId) { + public Builder setNativePluginId(final String nativePluginId) { checkNotNull(nativePluginId, "'nativePluginId' is null"); checkArgument(!nativePluginId.isEmpty(), "'nativePluginId' is empty"); checkArgument( @@ -545,17 +626,20 @@ public void setNativePluginId(final String nativePluginId) { || nativePluginId.equals("descriptor_set")), "'nativePluginId' matches one of the built-in protoc plugins"); this.nativePluginId = nativePluginId; + return this; } - public void setNativePluginExecutable(final String nativePluginExecutable) { + public Builder setNativePluginExecutable(final String nativePluginExecutable) { checkNotNull(nativePluginExecutable, "'nativePluginExecutable' is null"); this.nativePluginExecutable = nativePluginExecutable; + return this; } - public void setNativePluginParameter(final String nativePluginParameter) { + public Builder setNativePluginParameter(final String nativePluginParameter) { checkNotNull(nativePluginParameter, "'nativePluginParameter' is null"); checkArgument(!nativePluginParameter.contains(":"), "'nativePluginParameter' contains illegal characters"); this.nativePluginParameter = nativePluginParameter; + return this; } public Builder withDescriptorSetFile( @@ -570,6 +654,11 @@ public Builder withDescriptorSetFile( return this; } + public Builder useArgumentFile(final boolean useArgumentFile) { + this.useArgumentFile = useArgumentFile; + return this; + } + private void checkProtoFileIsInProtopath(final File protoFile) { assert protoFile.isFile(); checkState(checkProtoFileIsInProtopathHelper(protoFile.getParentFile())); @@ -668,7 +757,9 @@ public Protoc build() { pluginDirectory, nativePluginId, nativePluginExecutable, - nativePluginParameter); + nativePluginParameter, + tempDirectory, + useArgumentFile); } } }