Feature/wayback machine

functions/package.json

@@ -37,7 +37,7 @@
  "object.entries": "^1.0.4",
  "raven": "^2.6.0",
  "replaceall": "^0.1.6",
- "supports-color": "^5.4.0"
+ "xml2js": "^0.4.19"
  },
  "devDependencies": {
  "axios-mock-adapter": "^1.15.0",

functions/src/actions/wayback-machine.js

@@ -0,0 +1,136 @@
+// Third party imports
+const axios = require('axios');
+const mustache = require('mustache');
+const xml2js = require('xml2js');
+
+// Local imports
+const config = require('../config');
+const {debug} = require('../utils/logger')('ia:actions:wayback-machine');
+const dialog = require('../dialog');
+const endpointProcessor = require('../network/endpoint-processor');
+const traverse = require('../utils/traverse');
+const waybackStrings = require('../strings').intents.wayback;
+
+/**
+ * Handle wayback query action
+ * - fill slots of wayback query
+ * - perform data requests to archive and alexa rankings
+ * - construct response speech for action
+ *
+ * @param app
+ */
+function handler (app) {
+ // Create wayback object
+ let waybackObject = {
+ url: '',
+ earliestYear: 0,
+ latestYear: 0,
+ totalUniqueURLs: 0,
+ alexaWorldRank: 0,
+ alexaUSRank: 0,
+ speech: waybackStrings.default,
+ };
+
+ // Check to see that both parameters have content
+ if (!app.params.getByName('wayback') && !app.params.getByName('url')) {
+ debug('wayback action called by mistake');
+ dialog.ask(app, waybackObject);
+ }
+
+ // Get url parameter and make url queries
+ waybackObject.url = app.params.getByName('url');
+ const archiveQueryURL = endpointProcessor.preprocess(
+ config.wayback.ARCHIVE, app, waybackObject
+ );
+ const alexaQueryURL = endpointProcessor.preprocess(
+ config.wayback.ALEXA, app, waybackObject
+ );
+
+ return Promise.all([axios.get(archiveQueryURL), axios.get(alexaQueryURL)])
+ .then(function (requestData) {
+ return Promise.all([archiveEngine(requestData[0].data), xmlConverter(requestData[1].data)]);
+ })
+ .then(response => {
+ waybackObject = Object.assign(waybackObject, response[0]);
+ return alexaEngine(response[1]);
+ })
+ .then(response => {
+ waybackObject = Object.assign(waybackObject, response);
+
+ // Construct response dialog for action
+ if (waybackObject.alexaUSRank !== 0) {
+ waybackObject.speech = mustache.render(waybackStrings.speech, waybackObject);
+ waybackObject.speech += mustache.render(waybackStrings.additionalSpeech, waybackObject);
+ } else {
+ waybackObject.speech = mustache.render(waybackStrings.speech, waybackObject);
+ waybackObject.speech += '.';
+ }
+ dialog.close(app, waybackObject);
+ });
+} // End of handler
+
+function archiveEngine (archiveJSON) {
+ debug('Inside archiveEngine...');
+ const res = { earliestYear: 0, latestYear: 0, totalUniqueURLs: 0 };
+ // Create array of capture years and then find earliest year
+ // and most recent year.
+ let yearsArray = Object.keys(archiveJSON.captures);
+ res.earliestYear = yearsArray[0];
+ res.latestYear = yearsArray[yearsArray.length - 1];
+
+ // Traverse URL category
+
+ // Find baseline of URL count
+ res.totalUniqueURLs += traverse(archiveJSON.urls[res.earliestYear]);
+ // debug('Baseline url count: ' + obj.totalUniqueURLs);
+
+ res.totalUniqueURLs += traverse(archiveJSON.new_urls);
+ // debug('Final url count: ' + obj.totalUniqueURLs);
+
+ if (res.totalUniqueURLs > 0) {
+ debug('archiveEngine successful!');
+ return res;
+ } else {
+ let error = new Error('archiveEngine didn\'t work.');
+ return error;
+ }
+}
+
+function alexaEngine (alexaJSON) {
+ debug('Inside alexaEngine...');
+ const res = { alexaWorldRank: 0, alexaUSRank: 0 };
+ res.alexaWorldRank = alexaJSON['ALEXA']['SD'][0]['POPULARITY'][0]['$']['TEXT'];
+ try {
+ res.alexaUSRank = alexaJSON['ALEXA']['SD'][0]['COUNTRY'][0]['$']['RANK'];
+ } catch (e) {
+ debug('Country not found');
+ debug(e);
+ }
+ if (res.alexaWorldRank > 0) {
+ debug('alexaEngine successful!');
+ return res;
+ } else {
+ let error = new Error('alexaEngine didn\'t work.');
+ return error;
+ }
+}
+
+function xmlConverter (data) {
+ return new Promise(function (resolve, reject) {
+ debug('Inside xmlConverter promise...');
+ let XMLparser = new xml2js.Parser();
+ XMLparser.parseString(data, function (err, result) {
+ if (err) {
+ let error = new Error('The XML parser didn\'t work.');
+ reject(error);
+ } else {
+ debug('xmlConverter promise successful!');
+ resolve(JSON.parse(JSON.stringify(result)));
+ }
+ });
+ });
+}
+
+module.exports = {
+ handler,
+};

functions/src/config.js

@@ -31,6 +31,11 @@ module.exports = {
  DEFAULT_SONG_IMAGE: 'http://archive.org/images/notfound.png',
  },
 
+ wayback: {
+ ARCHIVE: 'http://web.archive.org/__wb/search/metadata?q={{url}}',
+ ALEXA: 'http://data.alexa.com/data?cli=10&url={{url}}',
+ },
+
  /**
  * settings specific for supported platforms
  */

functions/src/strings.js

@@ -444,6 +444,14 @@ module.exports = {
  speech: 'Version is {{version}}.',
  },
 
+ wayback: {
+ speech: '{{url}} was first captured by the Internet Archive in {{earliestYear}} and most recently in {{latestYear}}. The archive has {{totalUniqueURLs}} unique url\'s for this website. <break></break>{{url}} is ranked <say-as interpret-as="ordinal">{{alexaWorldRank}}</say-as> in the world in popularity',
+ additionalSpeech: ' and <say-as interpret-as="ordinal">{{alexaUSRank}}</say-as> in the United States.',
+ default: 'Would you like to use the wayback machine to hear the history of a website? Simply say Wayback Machine and the name of the website you\'d like to hear.',
+ error: "I'm sorry I'm having trouble here. Maybe we should try this again later.",
+ suggestions: ['wayback machine google.com', 'wayback machine archive.org']
+ },
+
  welcome: {
  acknowledges: [
  'Welcome to music at the Internet Archive.'

functions/src/utils/traverse.js

@@ -0,0 +1,31 @@
+const _ = require('lodash');
+// const {debug} = require('../utils/logger')('ia:actions:utils:traverse');
+
+/**
+ * Traverse a given object
+ *
+ * @param {Object} obj
+ */
+module.exports = function (obj) {
+ let results = [];
+ function traverse (obj) {
+ _.forOwn(obj, (val, key) => {
+ if (_.isArray(val)) {
+ val.forEach(el => {
+ traverse(el);
+ });
+ } else if (_.isObject(val)) {
+ traverse(val);
+ } else {
+ results.push(val);
+ }
+ });
+ }
+ traverse(obj);
+ let count = 0;
+ while (results.length !== 0) {
+ count += results.pop();
+ }
+ // debug('final count inside traverse = ' + count);
+ return count;
+};

functions/tests/actions/wayback-machine.spec.js

@@ -0,0 +1,58 @@
+const {expect} = require('chai');
+const rewire = require('rewire');
+
+const action = rewire('../../src/actions/wayback-machine');
+
+const archiveRequest = require('../fixtures/wayback-archive.json');
+const alexaRequest = require('../fixtures/wayback-alexa.xml');
+const mockApp = require('../_utils/mocking/platforms/app');
+const mockDialog = require('../_utils/mocking/dialog');
+
+describe('actions', () => {
+ describe('wayback machine', () => {
+ let app;
+ let dialog;
+
+ beforeEach(() => {
+ app = mockApp();
+ dialog = mockDialog();
+ action.__set__('dialog', dialog);
+ });
+
+ it('check to see that overall action eventually returns a promise', () => {
+ action.handler(app);
+ expect(Promise.resolve()).to.be.a('promise');
+ });
+
+ it('check to see that archiveEngine is working', () => {
+ let obj = { earliestYear: 0, latestYear: 0, totalUniqueURLs: 0 };
+ let result;
+
+ action.__set__('archiveEngine', function (a, b) {
+ result = action.archiveEngine(archiveRequest, obj);
+ expect(result).to.change(obj, 'earliestYear');
+ expect(result).to.change(obj, 'latestYear');
+ expect(result).to.change(obj, 'totalUniqueURLs');
+ });
+ });
+
+ it('check to see that alexaEngine is working', () => {
+ let obj = { alexaWorldRank: 0 };
+ let result;
+
+ action.__set__('alexaEngine', function (a, b) {
+ result = action.alexaEngine(alexaRequest, obj);
+ expect(result).to.change(obj, 'alexaWorldRank');
+ });
+ });
+
+ it('check to see that xmlConverter is working', () => {
+ let result;
+
+ action.__set__('xmlConverter', function (a) {
+ result = action.xmlConverter(alexaRequest);
+ expect(result).to.not.throw('The XML parser didn\'t work.');
+ });
+ });
+ });
+});

functions/tests/fixtures/wayback-alexa.xml

@@ -0,0 +1 @@
+<!-- Need more Alexa data? Find our APIs here: https://aws.amazon.com/alexa/ --><ALEXA VER="0.9" URL="cnn.com/" HOME="0" AID="=" IDN="cnn.com/"><SD><POPULARITY URL="cnn.com/" TEXT="104" SOURCE="panel"/><REACH RANK="88"/><RANK DELTA="-4"/><COUNTRY CODE="US" NAME="United States" RANK="27"/></SD></ALEXA>

functions/tests/utils/traverse.spec.js

@@ -0,0 +1,35 @@
+const {expect} = require('chai');
+const traverse = require('../../src/utils/traverse');
+
+let testJSON = {'captures': {
+ '1999': {
+ 'text/html': 18360
+ },
+ '2000': {
+ 'application/x-director': 19,
+ 'video/quicktime': 1584,
+ 'application/x-troff-man': 1,
+ 'x-world/x-vrml': 1,
+ 'audio/x-pn-realaudio': 176,
+ 'audio/mpeg': 195,
+ 'audio/x-wav': 3098,
+ 'image/png': 97,
+ 'text/html': 901401,
+ 'video/x-ms-asf': 142,
+ 'image/gif': 17388,
+ 'text/plain': 394428,
+ 'image/jpeg': 82903,
+ 'application/x-shockwave-flash': 39,
+ 'application/zip': 108,
+ 'audio/x-aiff': 2767,
+ 'text/css': 55,
+ 'application/pdf': 291
+ }}};
+
+describe('utils', () => {
+ describe('traverse', () => {
+ it('should traverse a given object to return the sum of it\'s leaf nodes', () => {
+ expect(traverse(testJSON)).to.be.equal(1423053);
+ });
+ });
+});