Checking out the diff between master and idio master

.travis.yml

@@ -0,0 +1,11 @@
+language: java
+jdk:
+- oraclejdk7
+install: mvn clean test package
+notifications:
+    hipchat:
+        rooms:
+            secure: HAIO6qjP1Os4yCduLwRfNrXP9K5v3hOpbXs/HOcoPzh0WACTNZUBJx8GPUWlcUJlzaxkUG05QC91dTn2kTyNteFE9xayKw347i7xywcYDttJdEL8M0agIkZowyrOfmiG+wFv/vCayZpf0T/MPYE1gDvFeP4yuP7CU0pdy1j0SRUkHvcpoXcx1OjbW7kMbiO1WedhzGZwrLWabw8UmNybvoyVSZmtBd4acRuOfzOfbynWoWL/9HD9jUoeCIbQeXWFxXGnQb22QpNWRYu7ZMH1ppkE/lI5zJtqZS5BjfEfK3tql4WaL72r4xO4uHkqi0xjUlP0iyQi4WW9G1PGFz6GmZSCrr04eyRQovhEaQcoces4+Q+uT/3pHplT12kE8y5wGTwYJlCfIRjGYN/uqfT6EEdl+8E4ZCA/o3t7rEHrRqxD98Pt+Q/y+GJxtYmGr4n+HAGCfa3BE4uij3N7EisO6mJ7NtMp5g8UGEvnqFV/eV46Urhoih4ILPhTkMKvX7PRxnSvrqE7toY9PXC5Ufkmt5TU7RmTNRT1cnH28MqlpeywWKNkNr8I6chfUCDNeEHT6Ckj00/l+CbQfNtQWQt7XApcCui0cxvTuZSrAKVzNTrWKcN/tMP5XyJ6t/bxBZAFDV4YHzK8fwzNHjz71sLGYukdvls4yLsEW74JqiQZ6Oo=
+        template:
+            - 'Build #%{build_number} (%{commit}) of %{repository_name}/%{branch} %{result} (%{duration}) %{build_url}'
+    email: false

README.md

@@ -5,26 +5,49 @@ json-wikipedia ![json-wikipedia](https://dl.dropboxusercontent.com/u/4663256/tmp
 
  - Please be aware that this tool does not work with the `multistream` dump.
 
-#### Setup ####
+ What's different about this fork:
+
+ - Uses Apache Spark to speedup the transformation to json. Original Json-wikipedia runs on a single thread.
+ - Fixes some issues with JWPL, which means less noisy extractions
+ - Chunks the article's pages into paragraphs and returns a list of links with correct spans
+ - Extract Links from article's paragraphs with matching spans
+
+
+## Convert the Wikipedia XML to JSON
+
+### The Docker way
+
+Enjoy a dockerized image:
+
+   `docker run -v <LOCALPATH>:/mnt -i -t dav009/jsonwikipedia  -input <PATHTOWIKI> -output <OUTPUTPATH> -lang <LANG> -action export-parallel`
 
-compile the project running 
+ For example if my `english_wikipedia.dump` lives in : `/david/data/english_wikipedia.dump` I could run it as:
 
-    mvn assembly:assembly 
-
-the command will produce a JAR file containing all the dependencies the target folder.  
+   `docker run -v /david/data:/mnt -i -t dav009/jsonwikipedia -input /mnt/english_wikipedia.dump -output /mnt/english_wikipedia.json -lang en -action export-parallel`
 
-#### Convert the Wikipedia XML to JSON ####
+Note that the output path corresponds to a path within the docker container. In the given example the output path is part of the mounted volume, so it will be available at the host machine.
 
-    java -cp target/json-wikipedia-1.0.0-jar-with-dependencies.jar it.cnr.isti.hpc.wikipedia.cli.MediawikiToJsonCLI -input wikipedia-dump.xml.bz -output wikipedia-dump.json[.gz] -lang [en|it] 		
 
-or 
+### Doing it yourself
 
-	./scripts/convert-xml-dump-to-json.sh [en|it] wikipedia-dump.xml.bz wikipedia-dump.json[.gz]
+1. Compile the project by doing: `mvn assembly:assembly` the command will produce a JAR file containing all the dependencies the target folder.
+2. Download Spark 1.3.1: http://www.apache.org/dyn/closer.cgi/spark/spark-1.3.1/spark-1.3.1.tgz
+3. Download Wikipedia Dump ( https://dumps.wikimedia.org/backup-index.html )
+4. Uncompress the Wikipedia Dump
+5. do:
 
-produces in `wikipedia-dump.json` the JSON version of the dump ([here you can find an example](https://dl.dropboxusercontent.com/u/4663256/tmp/json-wikipedia-sample.json)). Each line of the file contains an article 
-of dump encoded in JSON. Each JSON line can be deserialized in an [Article](http://sassicaia.isti.cnr.it/javadocs/json-wikipedia/it/cnr/isti/hpc/wikipedia/article/Article.html) object, 
-which represents an 
-_enriched_ version of the wikitext page. The Article object contains: 
+	SPARKFOLDER/bin/spark-submit --driver-memory 10G --class it.cnr.isti.hpc.wikipedia.cli.MediawikiToJsonCLI json-wikipedia-1.0.0-jar-with-dependencies.jar -input <PATHTODBPEDIADUMP> -output <PATHTONEWJSONPEDIA> -lang <LANG> -action export-parallel
+
+this produces in `<PATHTONEWJSONPEDIA>` the JSON version of the dump
+
+You can also call Jsonpedia the usual way but it will use a single thread to process the wiki:
+
+    java -cp target/json-wikipedia-1.0.0-jar-with-dependencies.jar it.cnr.isti.hpc.wikipedia.cli.MediawikiToJsonCLI -input wikipedia-dump.xml.bz -output wikipedia-dump.json[.gz] -lang [en|it] -action export
+
+### How does Jsonpedia look like?
+
+ ([here you can find an example](https://dl.dropboxusercontent.com/u/4663256/tmp/json-wikipedia-sample.json)). Each line of the file contains an article
+of dump encoded in JSON. Each JSON line can be deserialized in an [Article](http://sassicaia.isti.cnr.it/javadocs/json-wikipedia/it/cnr/isti/hpc/wikipedia/article/Article.html) object,which represents an _enriched_ version of the wikitext page. The Article object contains:
 
 
   * the title (e.g., Leonardo Da Vinci);
@@ -45,18 +68,18 @@ _enriched_ version of the wikitext page. The Article object contains:
   * a list of terms highlighted in the article;
   * if present, the infobox. 
 
-#### Usage ####
+#### Usage
 
 Once you have created (or downloaded) the JSON dump (say `wikipedia.json`), you can iterate over the articles of the collection 
 easily using this snippet: 
 
-    RecordReader<Article> reader = new RecordReader<Article>(
-			"wikipedia.json",new JsonRecordParser<Article>(Article.class)
-    ).filter(TypeFilter.STD_FILTER);
+	RecordReader<Article> reader = new RecordReader<Article>(
+	  "wikipedia.json",new JsonRecordParser<Article>(Article.class)
+	).filter(TypeFilter.STD_FILTER);
 
-    for (Article a : reader) {
-	// do what you want with your articles	
-    }
+	for (Article a : reader) {
+		// do what you want with your articles
+	}
 
 You can also add some filters in order to iterate only on certain articles (in the example 
 we used only the standard type filter, which excludes meta pages e.g., Portal: or User: pages.).
@@ -67,38 +90,38 @@ of the [hpc-utils](http://sassicaia.isti.cnr.it/javadocs/hpc-utils) package.
 
 In order to use these classes, you will have to install `json-wikipedia` in your maven repository:
 
-    mvn install
+	mvn install
 
 and import the project in your new maven project adding the dependency: 
 
-    <dependency>
-	    <groupId>it.cnr.isti.hpc</groupId>
+	<dependency>
+		<groupId>it.cnr.isti.hpc</groupId>
 		<artifactId>json-wikipedia</artifactId>
 		<version>1.0.0</version>
-	</dependency> 
-	
+	</dependency>
+  
 #### Schema ####
 
 ```
  |-- categories: array (nullable = true)
  |    |-- element: struct (containsNull = false)
- |    |    |-- description: string (nullable = true)
+ |    |    |-- anchor: string (nullable = true)
  |    |    |-- id: string (nullable = true)
  |-- externalLinks: array (nullable = true)
  |    |-- element: struct (containsNull = false)
- |    |    |-- description: string (nullable = true)
+ |    |    |-- anchor: string (nullable = true)
  |    |    |-- id: string (nullable = true)
  |-- highlights: array (nullable = true)
  |    |-- element: string (containsNull = false)
  |-- infobox: struct (nullable = true)
- |    |-- description: array (nullable = true)
+ |    |-- anchor: array (nullable = true)
  |    |    |-- element: string (containsNull = false)
  |    |-- name: string (nullable = true)
  |-- integerNamespace: integer (nullable = true)
  |-- lang: string (nullable = true)
  |-- links: array (nullable = true)
  |    |-- element: struct (containsNull = false)
- |    |    |-- description: string (nullable = true)
+ |    |    |-- anchor: string (nullable = true)
  |    |    |-- id: string (nullable = true)
  |-- lists: array (nullable = true)
  |    |-- element: array (containsNull = false)
@@ -119,7 +142,7 @@ and import the project in your new maven project adding the dependency:
  |    |    |    |    |-- element: string (containsNull = false)
  |-- templates: array (nullable = true)
  |    |-- element: struct (containsNull = false)
- |    |    |-- description: array (nullable = true)
+ |    |    |-- anchor: array (nullable = true)
  |    |    |    |-- element: string (containsNull = false)
  |    |    |-- name: string (nullable = true)
  |-- templatesSchema: array (nullable = true)