written, needs to be tested better.

.gitignore

@@ -1,2 +1,5 @@
 node_modules
 npm-debug.log
+.DS_Store
+examples/_*
+examples/*.json

Makefile

@@ -2,6 +2,7 @@
 test:
  @./node_modules/.bin/mocha \
  --require should \
- --reporter spec
+ --reporter spec \
+ --timeout 20s
 
-.PHONY: test
+.PHONY: test

Readme.md

@@ -3,6 +3,8 @@
 
  structure any website
 
+##
+
 ## License
 
 (The MIT License)

examples/dribbble-search.js

@@ -0,0 +1,65 @@
+/**
+ * Module Dependencies
+ */
+
+var xray = require('..');
+var write = require('fs').createWriteStream(__dirname + '/dribbble-search.json');
+
+/**
+ * Base URL
+ */
+
+var url = 'https://dribbble.com/search?q=form';
+
+/**
+ * x-ray
+ */
+
+xray(url)
+ .selector('.dribbbles > li')
+ .property('url', '.dribbble-img > .dribbble-link', 'href')
+ .property('name', '.dribbble-img img', 'alt')
+ .property('image', '.dribbble-img img', 'src')
+ .paginate('.next_page')
+ // .format(format)
+ .run(function(err, json) {
+ if (err) throw err;
+ // console.log(json);
+ // console.log(json);
+ // console.log('called');
+ // var str = JSON.stringify(arr, true, 2);
+ // write(__dirname + '/dribble.json', str, 'utf8');
+ // console.log('crawled!');
+ })
+ .end(function(err, json) {
+ console.log(err, json);
+ })
+ .stream(write);
+ // .end(function(err) {
+ // if (err) throw err;
+ // console.log('crawled');
+ // });
+
+/**
+ * Images
+ */
+
+function images($el, $) {
+ var imgs = [];
+
+ $el.find('a').each(function(i, el) {
+ imgs.push($(el).attr('data-src'));
+ })
+
+ return imgs;
+}
+
+/**
+ * Format
+ */
+
+function format(json) {
+ json.tags = json.tags.split(',');
+ json.shots = +json.shots.replace(/\D+/g, '');
+ json.followers = +json.followers.replace(/\D+/g, '');
+}

examples/github-stars.js

@@ -1,13 +1,71 @@
+/**
+ * Module Dependencies
+ */
 
-Structural
- .url('https:/stars/matthewmueller?direction=asc&language=javascript&sort=created')
- .url('https:/stars/matthewmueller?direction=desc&language=javascript&sort=created')
- .key('title', 'div > ul > li.repo-list-item.public.source > h3.repo-list-name > a')
+var xray = require('..');
+var array = require('array');
+var fmt = require('util').format;
+
+/**
+ * Urls
+ */
+
+var languages = [
+ 'JavaScript',
+ 'CSS',
+ 'CoffeeScript',
+ 'Ruby',
+ 'Python',
+ 'Shell',
+ 'Go',
+ 'C',
+ 'C++',
+ 'Objective-C',
+ 'PHP',
+ 'VimL',
+ 'Java',
+ 'Swift',
+ 'Scala',
+ 'TeX',
+ 'Perl',
+ 'Lua',
+ 'Clojure',
+ 'IDL',
+ 'Objective-C++',
+ 'Processing',
+ 'R',
+ 'Vala',
+ 'Lisp',
+ 'XSLT',
+ 'LiveScript',
+ 'TypeScript'
+];
+
+/**
+ * Get the urls
+ */
+
+var base = 'https:/stars/matthewmueller?direction=%s&language=%s&sort=created';
+var urls = [];
+languages.forEach(function(language) {
+ language = decodeURIComponent(language);
+ urls.push(fmt(base, 'asc', language));
+ urls.push(fmt(base, 'desc', language));
+})
+
+/**
+ * Use
+ */
+
+xray(urls)
+ .key('repo', 'div > ul > li.repo-list-item.public.source > h3.repo-list-name > a')
  .key('url', 'div > ul > li.repo-list-item.public.source > h3.repo-list-name > a', 'href')
  .key('author', 'ul > li.repo-list-item.public.source > h3.repo-list-name > a > span.prefix')
  .key('description', 'div > div > ul > li > p.repo-list-description')
+ .key('time', '.repo-list-meta time', 'datetime')
  .paginate('div.pagination > a:last-child')
  .json(function(err, json) {
  if (err) throw err;
- console.log(json);
+ json = array(json).unique('repo').toArray();
+ console.log(JSON.stringify(json, true, 2));
  })