add e2e tests for Spark Scala

doc/scala/Spark.ipynb

@@ -13,7 +13,7 @@
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
- "scrolled": true
+ "scrolled": false
  },
  "outputs": [],
  "source": [
@@ -61,6 +61,13 @@
  "source": [
  "spark.stop()"
  ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
  }
  ],
  "metadata": {

test/notebooks/scala/SparkScalaTest.ipynb

@@ -0,0 +1,72 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%%classpath add mvn\n",
+ "org.apache.spark spark-sql_2.11 2.2.1"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import org.apache.spark.sql.SparkSession\n",
+ "\n",
+ "val spark = SparkSession.builder()\n",
+ " .appName(\"Simple Application\")\n",
+ " .master(\"local[4]\")\n",
+ " .config(\"spark.ui.enabled\", \"false\")\n",
+ " .getOrCreate()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "val NUM_SAMPLES = 10000000\n",
+ "\n",
+ "val count = spark.sparkContext.parallelize(1 to NUM_SAMPLES).map{i =>\n",
+ " val x = Math.random()\n",
+ " val y = Math.random()\n",
+ " if (x*x + y*y < 1) 1 else 0\n",
+ "}.reduce(_ + _)\n",
+ "\n",
+ "println(\"Pi is roughly \" + 4.0 * count / NUM_SAMPLES)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "spark.stop()"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Scala",
+ "language": "scala",
+ "name": "scala"
+ },
+ "language_info": {
+ "codemirror_mode": "text/x-scala",
+ "file_extension": ".scala",
+ "mimetype": "",
+ "name": "Scala",
+ "nbconverter_exporter": "",
+ "version": "2.11.11"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

test/tests/beakerx.po.js

@@ -19,6 +19,10 @@ var BeakerXPageObject = function () {
  this.baseDocURL = 'http://127.0.0.1:8888/tree/doc/contents';
  this.baseTestURL = 'http://127.0.0.1:8888/tree/test/notebooks';
  this.kernelIdleIcon = $('i.kernel_idle_icon');
+ this.outputResultCss = 'div.output_subarea.output_text.output_result';
+ this.outputStderrCss = 'div.output_subarea.output_text.output_stderr';
+ this.outputStdoutCss = 'div.output_subarea.output_text.output_stdout';
+ this.allOutputTextCss = 'div.output_subarea.output_text';
 
  this.runNotebookByName = function(name, done, subDir){
  browser
@@ -134,7 +138,7 @@ var BeakerXPageObject = function () {
 
  this.runCellToGetOutputTextElement = function(index){
  var codeCell = this.runCodeCellByIndex(index);
- return codeCell.$('div.output_subarea.output_text');
+ return codeCell.$(this.allOutputTextCss);
  };
 
  this.plotLegendContainerIsEnabled = function(dtcontainer){
@@ -165,14 +169,35 @@ var BeakerXPageObject = function () {
  codeCell.scroll();
  var resultTest;
  try{
- resultTest = codeCell.$('div.output_subarea.output_text').getText();
+ resultTest = codeCell.$(this.allOutputTextCss).getText();
  }catch(e){
  console.log(expectedText + ' --- ' + e.toString());
  resultTest = this.runCellToGetOutputTextElement(index).getText();
  }
  expect(resultTest).toMatch(expectedText);
  };
 
+ this.waitAndCheckCellOutputStderrText = function(index, expectedText){
+ this.waitAndCheckCellOutputText(index, expectedText, this.outputStderrCss);
+ };
+
+ this.waitAndCheckCellOutputStdoutText = function(index, expectedText){
+ this.waitAndCheckCellOutputText(index, expectedText, this.outputStdoutCss);
+ };
+
+ this.waitAndCheckCellOutputResultText = function(index, expectedText){
+ this.waitAndCheckCellOutputText(index, expectedText, this.outputResultCss);
+ };
+
+ this.waitAndCheckCellOutputText = function(index, expectedText, selector){
+ var codeCell = this.getCodeCellByIndex(index);
+ codeCell.scroll();
+ browser.waitUntil(function () {
+ var output = codeCell.$(selector);
+ return output.isEnabled() && expectedText.test(output.getText());
+ }, 50000, 'expected output toMatch ' + expectedText);
+ };
+
  this.getTableColumnLabel = function(tableIndex, columnIndex){
  var table = $$("div.dataTables_scrollHead") [tableIndex];
  var tableColumnLabels = table.$$("span.header-text");

test/tests/groovy/groovyTest.js

@@ -76,12 +76,12 @@ describe('Groovy tests', function () {
  expect(str.charCodeAt(2).toString(16)).toEqual('44f');
  }
 
- describe('Cyrillic symbols', function () {
+ describe('Cyrillic symbols (Groovy) ', function () {
  var codeCell;
 
  it('Output contains UTF-8 hex string', function () {
  codeCell = beakerxPO.runCodeCellByIndex(7);
- expect(codeCell.$('.output_subarea.output_text').getText()).toMatch('d18dd18ed18f');
+ beakerxPO.waitAndCheckCellOutputStdoutText(7, /d18dd18ed18f/);
  });
 
  it('Plot title is cyrillic (cp1521)', function () {

test/tests/groovy/handlingCombinationOfCodeAndMagics.js

@@ -34,7 +34,7 @@ describe('Tests for combination of code and magics', function () {
 
  describe('Combination of code and magics', function () {
  it('mixing of println code and %time magic', function () {
- var outputs = beakerxPO.runCodeCellByIndex(cellIndex).$$('div.output_subarea.output_text');
+ var outputs = beakerxPO.runCodeCellByIndex(cellIndex).$$(beakerxPO.allOutputTextCss);
  browser.pause(2000);
  expect(outputs[0].getText()).toMatch(new RegExp('test1\n221\n' + timeExp.source));
  expect(outputs[0].getText()).toMatch(new RegExp('test2\n' + timeExp.source));
@@ -43,7 +43,7 @@ describe('Tests for combination of code and magics', function () {
 
  it('%import magic inside code', function () {
  cellIndex += 1;
- var outputs = beakerxPO.runCodeCellByIndex(cellIndex).$$('div.output_subarea.output_text');
+ var outputs = beakerxPO.runCodeCellByIndex(cellIndex).$$(beakerxPO.allOutputTextCss);
  browser.pause(2000);
  expect(outputs[0].getText()).toMatch(errorExp);
  expect(outputs[1].getText()).toMatch(new RegExp('221\n' + timeExp.source));

test/tests/scala/sparkScalaTest.js

@@ -0,0 +1,74 @@
+/*
+ * Copyright 2018 TWO SIGMA OPEN SOURCE, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var BeakerXPageObject = require('../beakerx.po.js');
+var beakerxPO;
+
+describe('Scala notebook', function () {
+
+ beforeAll(function () {
+ beakerxPO = new BeakerXPageObject();
+ beakerxPO.runNotebookByUrl('/notebooks/test/notebooks/scala/SparkScalaTest.ipynb');
+ });
+
+ afterAll(function () {
+ beakerxPO.closeAndHaltNotebook();
+ });
+
+ var cellIndex;
+
+ describe('Add Spark jars. ', function () {
+ it('Should add Spark jars', function () {
+ cellIndex = 0;
+ beakerxPO.runCodeCellByIndex(cellIndex);
+ beakerxPO.waitAndCheckCellOutputStdoutText(cellIndex, /spark-sql/);
+ beakerxPO.waitAndCheckCellOutputStdoutText(cellIndex, /spark-core/);
+ });
+ });
+
+ describe('Spark session. ', function () {
+ it('Spark should start session', function () {
+ cellIndex +=1;
+ beakerxPO.runCodeCellByIndex(cellIndex);
+ beakerxPO.waitAndCheckCellOutputStderrText(cellIndex, /Running Spark/);
+ beakerxPO.waitAndCheckCellOutputStderrText(cellIndex, /Successfully started service .sparkDriver./);
+ beakerxPO.waitAndCheckCellOutputStderrText(cellIndex,
+ /Successfully started service .org\.apache\.spark\.network\.netty\.NettyBlockTransferService./);
+ beakerxPO.waitAndCheckCellOutputResultText(cellIndex, /org.apache.spark.sql.SparkSession/);
+ });
+ });
+
+ describe('Spark job. ', function () {
+ it('Spark should calculate PI', function () {
+ cellIndex +=1;
+ beakerxPO.runCodeCellByIndex(cellIndex);
+ beakerxPO.waitAndCheckCellOutputStderrText(cellIndex, /Starting job/);
+ beakerxPO.waitAndCheckCellOutputStderrText(cellIndex, /Job \d* finished/);
+ beakerxPO.waitAndCheckCellOutputStdoutText(cellIndex, /Pi is roughly \d.\d*/)
+
+ });
+ });
+
+ describe('Spark stop. ', function () {
+ it('Spark should be stopped', function () {
+ cellIndex +=1;
+ beakerxPO.runCodeCellByIndex(cellIndex);
+ beakerxPO.waitAndCheckCellOutputStderrText(cellIndex, /MemoryStore cleared/);
+ beakerxPO.waitAndCheckCellOutputStderrText(cellIndex, /Successfully stopped SparkContext/);
+ });
+ });
+
+});