Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

#6569 maven cell magic #6665

Merged
merged 10 commits into from
Jan 24, 2018
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,16 @@
import com.twosigma.beakerx.kernel.magic.command.functionality.AddStaticImportMagicCommand;
import com.twosigma.beakerx.kernel.magic.command.functionality.BashMagicCommand;
import com.twosigma.beakerx.kernel.magic.command.functionality.ClasspathAddDynamicMagicCommand;
import com.twosigma.beakerx.kernel.magic.command.functionality.ClassPathAddMvnCellMagicCommand;
import com.twosigma.beakerx.kernel.magic.command.functionality.ClasspathAddJarMagicCommand;
import com.twosigma.beakerx.kernel.magic.command.functionality.ClasspathAddMvnMagicCommand;
import com.twosigma.beakerx.kernel.magic.command.functionality.ClasspathAddRepoMagicCommand;
import com.twosigma.beakerx.kernel.magic.command.functionality.ClasspathRemoveMagicCommand;
import com.twosigma.beakerx.kernel.magic.command.functionality.ClasspathShowMagicCommand;
import com.twosigma.beakerx.kernel.magic.command.functionality.HtmlAliasMagicCommand;
import com.twosigma.beakerx.kernel.magic.command.functionality.HtmlMagicCommand;
import com.twosigma.beakerx.kernel.magic.command.functionality.JavaScriptMagicCommand;
import com.twosigma.beakerx.kernel.magic.command.functionality.JSMagicCommand;
import com.twosigma.beakerx.kernel.magic.command.functionality.JavaScriptMagicCommand;
import com.twosigma.beakerx.kernel.magic.command.functionality.LoadMagicMagicCommand;
import com.twosigma.beakerx.kernel.magic.command.functionality.LsMagicCommand;
import com.twosigma.beakerx.kernel.magic.command.functionality.TimeCellModeMagicCommand;
Expand Down Expand Up @@ -57,6 +58,7 @@ public static List<MagicCommandType> createDefaults(KernelFunctionality kernel)
lsmagic(magicCommandTypes),
addJar(kernel),
addJarByMvn(kernel),
addJarByMvnCell(kernel),
addDynamic(kernel),
addRepo(kernel),
removeJar(kernel),
Expand Down Expand Up @@ -123,6 +125,13 @@ private static MagicCommandType addJarByMvn(KernelFunctionality kernel) {
kernel.getTempFolder().toString() + MVN_DIR), kernel));
}

private static MagicCommandType addJarByMvnCell(KernelFunctionality kernel) {
return new MagicCommandType(ClassPathAddMvnCellMagicCommand.CLASSPATH_ADD_MVN_CELL, "<group name version>",
new ClassPathAddMvnCellMagicCommand(new MavenJarResolver.ResolverParams(
kernel.getCacheFolder().toString() + "/maven/cache",
kernel.getTempFolder().toString() + MVN_DIR), kernel));
}

private static MagicCommandType addJar(KernelFunctionality kernel) {
return new MagicCommandType(ClasspathAddJarMagicCommand.CLASSPATH_ADD_JAR, "<jar path>", new ClasspathAddJarMagicCommand(kernel));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,27 +15,26 @@
*/
package com.twosigma.beakerx.kernel.magic.command;

import static com.google.common.base.Preconditions.checkNotNull;

import com.google.common.base.Preconditions;
import com.twosigma.beakerx.kernel.commands.MavenInvocationSilentOutputHandler;
import com.twosigma.beakerx.kernel.commands.MavenJarResolverSilentLogger;
import com.twosigma.beakerx.kernel.magic.command.functionality.MvnLoggerWidget;
import org.apache.commons.io.FileUtils;
import org.apache.maven.shared.invoker.DefaultInvocationRequest;
import org.apache.maven.shared.invoker.DefaultInvoker;
import org.apache.maven.shared.invoker.InvocationRequest;
import org.apache.maven.shared.invoker.InvocationResult;
import org.apache.maven.shared.invoker.Invoker;

import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.UUID;

import com.twosigma.beakerx.kernel.magic.command.functionality.MvnLoggerWidget;
import org.apache.commons.io.FileUtils;
import org.apache.maven.shared.invoker.DefaultInvocationRequest;
import org.apache.maven.shared.invoker.DefaultInvoker;
import org.apache.maven.shared.invoker.InvocationRequest;
import org.apache.maven.shared.invoker.InvocationResult;
import org.apache.maven.shared.invoker.Invoker;

public class MavenJarResolver {

public static final String MVN_DIR = File.separator + "mvnJars";
Expand All @@ -49,17 +48,21 @@ public class MavenJarResolver {

public MavenJarResolver(final ResolverParams commandParams,
PomFactory pomFactory) {
this.commandParams = checkNotNull(commandParams);
this.commandParams = Preconditions.checkNotNull(commandParams);
this.pathToMavenRepo = getOrCreateFile(commandParams.getPathToNotebookJars()).getAbsolutePath();
this.pomFactory = pomFactory;
}

public AddMvnCommandResult retrieve(String groupId, String artifactId, String version, MvnLoggerWidget progress) {
List<Dependency> dependencies = Arrays.asList(new Dependency(groupId, artifactId, version));
return retrieve(dependencies, progress);
}

public AddMvnCommandResult retrieve(List<Dependency> dependencies, MvnLoggerWidget progress) {
File finalPom = null;
try {
Dependency dependency = new Dependency(groupId, artifactId, version);
String pomAsString = pomFactory.createPom(pathToMavenRepo, dependency, commandParams.getRepos());
finalPom = saveToFile(commandParams.getPathToNotebookJars(), dependency, pomAsString);
String pomAsString = pomFactory.createPom(pathToMavenRepo, dependencies, commandParams.getRepos());
finalPom = saveToFile(commandParams.getPathToNotebookJars(), pomAsString);
InvocationRequest request = createInvocationRequest();
request.setOffline(commandParams.getOffline());
request.setPomFile(finalPom);
Expand All @@ -68,18 +71,17 @@ public AddMvnCommandResult retrieve(String groupId, String artifactId, String ve
progress.display();
InvocationResult invocationResult = invoker.execute(request);
progress.close();
return getResult(invocationResult, groupId, artifactId, version);
return getResult(invocationResult, dependencies);
} catch (Exception e) {
return AddMvnCommandResult.error(e.getMessage());
} finally {
deletePomFolder(finalPom);
}
}

private File saveToFile(String pathToNotebookJars, Dependency dependency, String pomAsString)
private File saveToFile(String pathToNotebookJars, String pomAsString)
throws IOException {
File finalPom = new File(pathToNotebookJars + "/poms/pom-" + UUID.randomUUID() + "-" +
dependency.getGroupId() + dependency.getArtifactId() + dependency.getVersion() + "xml");
File finalPom = new File(pathToNotebookJars + "/poms/pom-" + UUID.randomUUID() + "-" + "xml");

FileUtils.writeStringToFile(finalPom, pomAsString, StandardCharsets.UTF_8);
return finalPom;
Expand Down Expand Up @@ -115,12 +117,19 @@ public String findMvn() {
return mavenLocation;
}

private AddMvnCommandResult getResult(InvocationResult invocationResult, String groupId, String artifactId, String version) {
private AddMvnCommandResult getResult(InvocationResult invocationResult, List<Dependency> dependencies) {
if (invocationResult.getExitCode() != 0) {
if (invocationResult.getExecutionException() != null) {
return AddMvnCommandResult.error(invocationResult.getExecutionException().getMessage());
}
return AddMvnCommandResult.error("Could not resolve dependencies for: " + groupId + " : " + artifactId + " : " + version);
StringBuilder errorMsgBuilder = new StringBuilder("Could not resolve dependencies for:");
for (Dependency dependency : dependencies) {
errorMsgBuilder
.append("\n").append(dependency.groupId).append(" : ")
.append(dependency.artifactId).append(" : ")
.append(dependency.version);
}
return AddMvnCommandResult.error(errorMsgBuilder.toString());
}

return AddMvnCommandResult.SUCCESS;
Expand Down Expand Up @@ -207,8 +216,8 @@ public static class ResolverParams {
private Map<String, String> repos = Collections.emptyMap();

public ResolverParams(String pathToCache, String pathToNotebookJars, boolean offline) {
this.pathToCache = checkNotNull(pathToCache);
this.pathToNotebookJars = checkNotNull(pathToNotebookJars);
this.pathToCache = Preconditions.checkNotNull(pathToCache);
this.pathToNotebookJars = Preconditions.checkNotNull(pathToNotebookJars);
this.offline = offline;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,38 +15,42 @@
*/
package com.twosigma.beakerx.kernel.magic.command;

import static com.twosigma.beakerx.kernel.magic.command.MavenJarResolver.POM_XML;

import com.twosigma.beakerx.kernel.magic.command.MavenJarResolver.Dependency;
import org.apache.commons.io.IOUtils;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;

import org.apache.commons.io.IOUtils;

public class PomFactory {

public String createPom(String pathToMavenRepo, Dependency dependency, Map<String, String> repos) throws IOException {
InputStream pom = getClass().getClassLoader().getResourceAsStream(POM_XML);
public String createPom(String pathToMavenRepo, List<Dependency> dependencies, Map<String, String> repos) throws IOException {
InputStream pom = getClass().getClassLoader().getResourceAsStream(MavenJarResolver.POM_XML);
String pomAsString = IOUtils.toString(pom, StandardCharsets.UTF_8);
pomAsString = configureOutputDir(pathToMavenRepo, pomAsString);
pomAsString = configureDependencies(dependency.getGroupId(), dependency.getArtifactId(), dependency.getVersion(), pomAsString);
pomAsString = configureDependencies(dependencies, pomAsString);
pomAsString = configureRepos(repos, pomAsString);
return pomAsString;
}

private String configureDependencies(String groupId, String artifactId, String version, String pomAsString) {
private String configureDependencies(List<Dependency> dependencies, String pomAsString) {
for (Dependency dependency : dependencies) {
pomAsString = configureDependency(dependency, pomAsString);
}
return pomAsString;
}

private String configureDependency(Dependency dependency, String pomAsString) {
return pomAsString.replace(
"<dependencies></dependencies>",
"<dependencies>\n" +
"</dependencies>",
" <dependency>\n" +
" <groupId>" + groupId + "</groupId>\n" +
" <artifactId>" + artifactId + "</artifactId>\n" +
" <version>" + version + "</version>\n" +
" <groupId>" + dependency.groupId + "</groupId>\n" +
" <artifactId>" + dependency.artifactId + "</artifactId>\n" +
" <version>" + dependency.version + "</version>\n" +
" </dependency>\n" +
"</dependencies>");
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
/*
* Copyright 2017 TWO SIGMA OPEN SOURCE, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*import static org.assertj.core.api.Assertions.assertThat;
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.twosigma.beakerx.kernel.magic.command.functionality;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

all files need banner


import com.twosigma.beakerx.kernel.KernelFunctionality;
import com.twosigma.beakerx.kernel.magic.command.MagicCommandExecutionParam;
import com.twosigma.beakerx.kernel.magic.command.MavenJarResolver;
import com.twosigma.beakerx.kernel.magic.command.PomFactory;
import com.twosigma.beakerx.kernel.magic.command.outcome.MagicCommandOutcomeItem;
import com.twosigma.beakerx.kernel.magic.command.outcome.MagicCommandOutput;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

public class ClassPathAddMvnCellMagicCommand extends ClasspathMagicCommand{

public static final String ADD = "add";
public static final String MVN = "mvn";
public static final String CLASSPATH_ADD_MVN_CELL = "%" + CLASSPATH + " " + ADD + " " + MVN;
public static final String MVN_CELL_FORMAT_ERROR_MESSAGE =
"Wrong command format, should be " + CLASSPATH_ADD_MVN_CELL + "\n"
+ " group name version or group:name:version" + "\n"
+ " group name version or group:name:version";
private static final String SPLIT_LINE_REGEX = "\\r?\\n";

private MavenJarResolver.ResolverParams commandParams;
private PomFactory pomFactory;

public ClassPathAddMvnCellMagicCommand(MavenJarResolver.ResolverParams commandParams, KernelFunctionality kernel) {
super(kernel);
this.commandParams = commandParams;
this.pomFactory = new PomFactory();
}

@Override
public String getMagicCommandName() {
return CLASSPATH_ADD_MVN_CELL;
}

@Override
public boolean matchCommand(String command) {
String[] commandParts = MagicCommandUtils.splitPath(command);
return commandParts.length >= 3
&& commandParts[0].equals("%" + CLASSPATH)
&& commandParts[1].equals(ADD)
&& commandParts[2].equals(MVN);
}

@Override
public MagicCommandOutcomeItem execute(MagicCommandExecutionParam param) {
String command = param.getCommand();
String commandCodeBlock = param.getCommandCodeBlock();
if (commandCodeBlock != null){
command += "\n" + commandCodeBlock;
}
String[] commandLines = command.split(SPLIT_LINE_REGEX);
unifyMvnLineFormat(commandLines);
if (!validateCommandLines(commandLines)){
return new MagicCommandOutput(MagicCommandOutput.Status.ERROR, MVN_CELL_FORMAT_ERROR_MESSAGE);
}
List<MavenJarResolver.Dependency> dependencies =
getDepsFromCommand(Arrays.copyOfRange(commandLines, 1, commandLines.length));
MavenJarResolver mavenJarResolver = new MavenJarResolver(commandParams, pomFactory);
MvnLoggerWidget mvnLoggerWidget = new MvnLoggerWidget(param.getCode().getMessage());
MavenJarResolver.AddMvnCommandResult result = mavenJarResolver.retrieve(dependencies, mvnLoggerWidget);

if (result.isJarRetrieved()) {
return handleAddedJars(mavenJarResolver.getPathToMavenRepo() + "/*");
}
return new MagicCommandOutput(MagicCommandOutput.Status.ERROR, result.getErrorMessage());
}

private List<MavenJarResolver.Dependency> getDepsFromCommand(String[] lines) {
List<MavenJarResolver.Dependency> dependencies = new ArrayList<>();
for (String line : lines) {
String[] dependencyData = MagicCommandUtils.splitPath(line);
dependencies.add(new MavenJarResolver.Dependency(dependencyData[0], dependencyData[1], dependencyData[2]));
}
return dependencies;
}

private boolean validateCommandLines(String[] commandLines) {
boolean isValid = false;
for (int i = 1; i < commandLines.length; i++) {
if (MagicCommandUtils.splitPath(commandLines[i]).length == 3){
isValid = true;
} else {
isValid = false;
break;
}
}
return isValid;
}

private void unifyMvnLineFormat(String[] mvnLines){
for (int i = 0; i < mvnLines.length; i++) {
mvnLines[i] = mvnLines[i].replace(":"," ");
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,6 @@
import com.twosigma.beakerx.kernel.magic.command.outcome.MagicCommandOutcomeItem;
import com.twosigma.beakerx.kernel.magic.command.outcome.MagicCommandOutput;

import java.util.Collection;

import static com.twosigma.beakerx.kernel.magic.command.functionality.MagicCommandUtils.splitPath;

public class ClasspathAddJarMagicCommand extends ClasspathMagicCommand {
Expand Down Expand Up @@ -59,12 +57,7 @@ public MagicCommandOutcomeItem execute(MagicCommandExecutionParam param) {
if (errorData.hasError()) {
return new MagicCommandOutput(MagicCommandOutput.Status.ERROR, errorData.getMessage());
} else {
Collection<String> newAddedJars = addJars(path);
if (newAddedJars.isEmpty()) {
return new MagicCommandOutput(MagicCommandOutput.Status.OK);
}
String textMessage = "Added " + JAR + (newAddedJars.size() > 1 ? "s: " : ": ") + newAddedJars;
return new MagicCommandOutput(MagicCommandOutput.Status.OK, textMessage);
return handleAddedJars(path);
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,16 +23,12 @@
import com.twosigma.beakerx.kernel.magic.command.outcome.MagicCommandOutcomeItem;
import com.twosigma.beakerx.kernel.magic.command.outcome.MagicCommandOutput;

import java.util.Collection;

import static com.twosigma.beakerx.kernel.magic.command.functionality.MagicCommandUtils.splitPath;

public class ClasspathAddMvnMagicCommand extends ClasspathMagicCommand {

public static final String ADD = "add";
public static final String MVN = "mvn";
public static final String CLASSPATH_ADD_MVN = CLASSPATH + " " + ADD + " " + MVN;
public static final String ADD_MVN_FORMAT_ERROR_MESSAGE = "Wrong command format, should be " + CLASSPATH_ADD_MVN + " group name version or " + CLASSPATH_ADD_MVN + " group:name:version";
public static final String ADD_MVN_FORMAT_ERROR_MESSAGE = "Wrong command format, should be" + CLASSPATH_ADD_MVN + " group name version or " + CLASSPATH_ADD_MVN + " group:name:version";

private MavenJarResolver.ResolverParams commandParams;
private PomFactory pomFactory;
Expand All @@ -57,7 +53,7 @@ public boolean matchCommand(String command) {
@Override
public MagicCommandOutcomeItem execute(MagicCommandExecutionParam param) {
String command = param.getCommand();
String[] split = splitPath(command);
String[] split = MagicCommandUtils.splitPath(command);
if (split.length != 6 && split.length != 4) {
return new MagicCommandOutput(MagicCommandOutput.Status.ERROR, ADD_MVN_FORMAT_ERROR_MESSAGE);
}
Expand All @@ -77,12 +73,7 @@ public MagicCommandOutcomeItem execute(MagicCommandExecutionParam param) {
}

if (result.isJarRetrieved()) {
Collection<String> newAddedJars = addJars(classpathAddMvnCommand.getPathToMavenRepo() + "/*");
if (newAddedJars.isEmpty()) {
return new MagicCommandOutput(MagicCommandOutput.Status.OK);
}
String textMessage = "Added jar" + (newAddedJars.size() > 1 ? "s: " : ": ") + newAddedJars;
return new MagicCommandOutput(MagicCommandOutput.Status.OK, textMessage);
return handleAddedJars(classpathAddMvnCommand.getPathToMavenRepo() + "/*");
}
return new MagicCommandOutput(MagicCommandOutput.Status.ERROR, result.getErrorMessage());
}
Expand Down
Loading