Feature/wayback machine

functions/package.json

@@ -37,7 +37,7 @@
  "object.entries": "^1.0.4",
  "raven": "^2.6.0",
  "replaceall": "^0.1.6",
- "supports-color": "^5.4.0"
+ "xml2js": "^0.4.19"
  },
  "devDependencies": {
  "axios-mock-adapter": "^1.15.0",

functions/src/actions/wayback-machine.js

@@ -0,0 +1,122 @@
+// Third party imports
+const axios = require('axios');
+const mustache = require('mustache');
+const xml2js = require('xml2js');
+
+// Local imports
+const config = require('../config');
+const {debug} = require('../utils/logger')('ia:actions:wayback-machine');
+const dialog = require('../dialog');
+const endpointProcessor = require('../network/endpoint-processor');
+const traverse = require('../utils/traverse');
+const waybackStrings = require('../strings').intents.wayback;
+
+/**
+ * Handle wayback query action
+ * - fill slots of wayback query
+ * - perform data requests to archive and alexa rankings
+ * - construct response speech for action
+ *
+ * @param app
+ */
+function handler (app) {
+ // Create wayback object
+ const waybackObject = {
+ url: '',
+ earliestYear: 0,
+ latestYear: 0,
+ totalUniqueURLs: 0,
+ alexaWorldRank: 0,
+ alexaUSRank: 0,
+ speech: waybackStrings.default,
+ };
+
+ // Check to see that both parameters have content
+ if (!app.params.getByName('wayback') && !app.params.getByName('url')) {
+ debug('wayback action called by mistake');
+ dialog.ask(app, waybackObject);
+ }
+
+ // Get url parameter and make url queries
+ waybackObject.url = app.params.getByName('url');
+ const archiveQueryURL = endpointProcessor.preprocess(
+ config.wayback.ARCHIVE, app, waybackObject
+ );
+ const alexaQueryURL = endpointProcessor.preprocess(
+ config.wayback.ALEXA, app, waybackObject
+ );
+
+ return Promise.all([axios.get(archiveQueryURL), axios.get(alexaQueryURL)])
+ .then(function (allData) {
+ // All data available here in the order it was called.
+
+ // Parse data from archive request
+ let archiveJSON = allData[0].data;
+ archiveEngine(archiveJSON, waybackObject);
+
+ // Parse data from alexa request
+ let XMLparser = new xml2js.Parser();
+ let convertXML = new Promise((resolve, reject) => {
+ XMLparser.parseString(allData[1].data, function (err, result) {
+ if (err) {
+ let error = new Error('The XML parser didn\'t work. Error message: ' + err);
+ reject(error);
+ } else {
+ resolve(result);
+ }
+ });
+ });
+ convertXML
+ .then(function (fulfilled) {
+ debug('XML parse successful!');
+ alexaEngine(JSON.parse(JSON.stringify(fulfilled)), waybackObject);
+ })
+ .catch(function (error) {
+ debug(error.message);
+ waybackObject.speech = waybackStrings.error;
+ dialog.ask(app, waybackObject);
+ });
+
+ // Construct response dialog for action
+ if (waybackObject.alexaUSRank !== 0) {
+ waybackObject.speech = mustache.render(waybackStrings.speech, waybackObject);
+ waybackObject.speech += mustache.render(waybackStrings.additionalSpeech, waybackObject);
+ } else {
+ waybackObject.speech = mustache.render(waybackStrings.speech, waybackObject);
+ waybackObject.speech += '.';
+ }
+
+ dialog.close(app, waybackObject);
+ });
+} // End of handler
+
+function archiveEngine (archiveJSON, waybackObject) {
+ // Create array of capture years and then find earliest year
+ // and most recent year.
+ let yearsArray = Object.keys(archiveJSON.captures);
+ waybackObject.earliestYear = yearsArray[0];
+ waybackObject.latestYear = yearsArray[yearsArray.length - 1];
+
+ // Traverse URL category
+
+ // Find baseline of URL count
+ waybackObject.totalUniqueURLs += traverse(archiveJSON.urls[waybackObject.earliestYear]);
+ // debug('Baseline url count: ' + waybackObject.totalUniqueURLs);
+
+ waybackObject.totalUniqueURLs += traverse(archiveJSON.new_urls);
+ // debug('Final url count: ' + waybackObject.totalUniqueURLs);
+}
+
+function alexaEngine (alexaJSON, waybackObject) {
+ waybackObject.alexaWorldRank = alexaJSON['ALEXA']['SD'][0]['POPULARITY'][0]['$']['TEXT'];
+ try {
+ waybackObject.alexaUSRank = alexaJSON['ALEXA']['SD'][0]['COUNTRY'][0]['$']['RANK'];
+ } catch (e) {
+ debug('Country not found');
+ debug(e);
+ }
+}
+
+module.exports = {
+ handler,
+};

functions/src/config.js

@@ -31,6 +31,11 @@ module.exports = {
  DEFAULT_SONG_IMAGE: 'http://archive.org/images/notfound.png',
  },
 
+ wayback: {
+ ARCHIVE: 'http://web.archive.org/__wb/search/metadata?q={{url}}',
+ ALEXA: 'http://data.alexa.com/data?cli=10&url={{url}}',
+ },
+
  /**
  * settings specific for supported platforms
  */

functions/src/strings.js

@@ -444,6 +444,14 @@ module.exports = {
  speech: 'Version is {{version}}.',
  },
 
+ wayback: {
+ speech: '{{url}} was first captured by the Internet Archive in {{earliestYear}} and most recently in {{latestYear}}. The archive has {{totalUniqueURLs}} unique url\'s for this website. <break></break>{{url}} is ranked <say-as interpret-as="ordinal">{{alexaWorldRank}}</say-as> in the world in popularity',
+ additionalSpeech: ' and <say-as interpret-as="ordinal">{{alexaUSRank}}</say-as> in the United States.',
+ default: 'Would you like to use the wayback machine to hear the history of a website? Simply say Wayback Machine and the name of the website you\'d like to hear.',
+ error: "I'm sorry I'm having trouble here. Maybe we should try this again later.",
+ suggestions: ['wayback machine google.com', 'wayback machine archive.org']
+ },
+
  welcome: {
  acknowledges: [
  'Welcome to music at the Internet Archive.'

functions/src/utils/traverse.js

@@ -0,0 +1,31 @@
+const _ = require('lodash');
+const {debug} = require('../utils/logger')('ia:actions:utils:traverse');
+
+/**
+ * Traverse a given object
+ *
+ * @param {Object} obj
+ */
+module.exports = function (obj) {
+ let results = [];
+ function traverse (obj) {
+ _.forOwn(obj, (val, key) => {
+ if (_.isArray(val)) {
+ val.forEach(el => {
+ traverse(el);
+ });
+ } else if (_.isObject(val)) {
+ traverse(val);
+ } else {
+ results.push(val);
+ }
+ });
+ }
+ traverse(obj);
+ let count = 0;
+ while (results.length !== 0) {
+ count += results.pop();
+ }
+ debug('final count inside traverse = ' + count);
+ return count;
+};

functions/tests/actions/wayback-machine.spec.js

@@ -0,0 +1,25 @@
+const {expect} = require('chai');
+const rewire = require('rewire');
+
+const action = rewire('../../src/actions/wayback-machine');
+
+const mockApp = require('../_utils/mocking/platforms/app');
+const mockDialog = require('../_utils/mocking/dialog');
+
+describe('actions', () => {
+ describe('wayback machine', () => {
+ let app;
+ let dialog;
+
+ beforeEach(() => {
+ app = mockApp();
+ dialog = mockDialog();
+ action.__set__('dialog', dialog);
+ });
+
+ it('check to see that a promise is returned with network requests', () => {
+ action.handler(app);
+ expect(Promise.resolve()).to.be.a('promise');
+ });
+ });
+});

functions/tests/utils/traverse.spec.js

@@ -0,0 +1,35 @@
+const {expect} = require('chai');
+const traverse = require('../../src/utils/traverse');
+
+let testJSON = {'captures': {
+ '1999': {
+ 'text/html': 18360
+ },
+ '2000': {
+ 'application/x-director': 19,
+ 'video/quicktime': 1584,
+ 'application/x-troff-man': 1,
+ 'x-world/x-vrml': 1,
+ 'audio/x-pn-realaudio': 176,
+ 'audio/mpeg': 195,
+ 'audio/x-wav': 3098,
+ 'image/png': 97,
+ 'text/html': 901401,
+ 'video/x-ms-asf': 142,
+ 'image/gif': 17388,
+ 'text/plain': 394428,
+ 'image/jpeg': 82903,
+ 'application/x-shockwave-flash': 39,
+ 'application/zip': 108,
+ 'audio/x-aiff': 2767,
+ 'text/css': 55,
+ 'application/pdf': 291
+ }}};
+
+describe('utils', () => {
+ describe('traverse', () => {
+ it('should traverse a given object to return the sum of it\'s leaf nodes', () => {
+ expect(traverse(testJSON)).to.be.equal(1423053);
+ });
+ });
+});