first commit

Author: vdavez

.gitignore

@@ -0,0 +1 @@
+node_modules

package.json

@@ -0,0 +1,17 @@
+{
+  "name": "dcmr",
+  "version": "0.0.0",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "license": "BSD",
+  "description": "ERROR: No README.md file found!",
+  "dependencies": {
+    "cheerio": "~0.10.1",
+    "get": "~1.2.1",
+    "underscore": "~1.4.1",
+    "url":"*",
+    "request": "~2.11.4"
+  }
+}

readme.md

@@ -0,0 +1,11 @@
+# DCMR
+
+Scraper borrowed from https://github.com/tmcw/dcmr. 
+
+To come:
+ 
+>> Parser into [Code XML](https://github.com/JoshData/dc-code-prototype) Schema.
+
+>> Simple-generator
+
+>> SimpleDCMR to come.

scrape/scrape.js

@@ -0,0 +1,90 @@
+#!/usr/bin/env node
+
+var cheerio = require('cheerio'),
+    fs = require('fs'),
+    url = require('url'),
+    get = require('get'),
+    _ = require('underscore'),
+    request = require('request');
+
+function start() {
+    var u = 'http://www.dcregs.dc.gov/Search/DCMRSearchByTitle.aspx';
+    request(u, function(err, reponse, body) {
+        if (err) throw err;
+
+        var $ = cheerio.load(body);
+        var togo = [];
+        var links = $('a').each(function(i, a) {
+            if (a.attribs.href.match(/\/Gateway\/TitleHome\.aspx\?/g)) {
+                togo.push(url.resolve(u, a.attribs.href));
+            }
+        });
+
+        togo.map(chapterhome);
+    });
+}
+
+function chapterhome(u) {
+    request(u, function(err, reponse, body) {
+        if (err) throw err;
+        console.log('to chapter step');
+
+        var $ = cheerio.load(body);
+        var togo = [];
+        var links = $('a').each(function(i, a) {
+            if (a.attribs.href.match(/ChapterHome\.aspx\?/)) {
+                togo.push(url.resolve(u, a.attribs.href));
+            }
+        });
+
+        togo.map(rulehome);
+    });
+}
+
+function rulehome(u) {
+    request(u, function(err, reponse, body) {
+        if (err) throw err;
+
+        console.log('to rule step');
+        var $ = cheerio.load(body);
+        var togo = [];
+        var links = $('a').each(function(i, a) {
+            if (a.attribs.href.match(/RuleHome\.aspx\?/)) {
+                togo.push(url.resolve(u, a.attribs.href));
+            }
+        });
+
+        togo.map(download);
+    });
+}
+
+function download(u) {
+    request(u, function(err, reponse, body) {
+        if (err) throw err;
+
+        console.log('to dl step');
+        var $ = cheerio.load(body);
+        var togo = [];
+        var links = $('a').each(function(i, a) {
+            // LOL
+            if (a.attribs.href && a.attribs.href.match(/Download\.aspx\?/)) {
+                var title = $('title').text().replace(/\r\n/, '');
+                togo.push([title, url.resolve(u, a.attribs.href)]);
+            }
+        });
+
+        togo.map(dl);
+    });
+}
+
+var i = 0;
+
+function dl(u) {
+    var id = i++;
+    get(u[1]).toDisk('docs/' + i + '.doc', function(err) {
+        if (err) console.log(err);
+    });
+    fs.writeFileSync('docs/' + i + '.title', u[0]);
+}
+
+start();