Skip to content
This repository has been archived by the owner on Sep 21, 2021. It is now read-only.

Commit

Permalink
port to Lucene 4.6. Tested against the solr-4.6 index of exampledocs …
Browse files Browse the repository at this point in the history
…xml files.

Changelist:
1. bumped up the version of maven-assembly plugin to the latest
2. added maven-jar plugin which copies all binary dependencies into lib/ subdirectory
3. Fixed AnalyzerToolPlugin to use token stream reset outside while loop over tokens
4. added lock factory handling in FsDirectory (untested)
5. updated trivia in about.xml
  • Loading branch information
DmitryKey committed Dec 5, 2013
1 parent 595f160 commit 87e7d66
Show file tree
Hide file tree
Showing 7 changed files with 101 additions and 72 deletions.
38 changes: 35 additions & 3 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,15 @@

<groupId>luke</groupId>
<artifactId>luke</artifactId>
<version>4.5.0</version>
<version>4.6.0</version>

<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<lucene.version>4.5.0</lucene.version>
<lucene.version>4.6.0</lucene.version>
<ehcache.version>2.6.2</ehcache.version>
<hadoop.version>0.20.2</hadoop.version>
<log4j.version>1.2.17</log4j.version>
<maven-assembly-plugin.version>2.3</maven-assembly-plugin.version>
<maven-assembly-plugin.version>2.4</maven-assembly-plugin.version>
<rhino.version>1.7R4</rhino.version>

</properties>
Expand Down Expand Up @@ -149,6 +149,38 @@
</configuration>
</plugin>

<!-- create a tar.gz distribution file with all the binary dependencies in lib directory -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<configuration>
<archive>
<manifest>
<addClasspath>true</addClasspath>
<classpathPrefix>lib/</classpathPrefix>
<mainClass>org.getopt.luke.Luke</mainClass>
</manifest>
<manifestEntries>
<Class-Path>.</Class-Path>
</manifestEntries>
</archive>
</configuration>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/lib</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>

<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<version>${maven-assembly-plugin.version}</version>
Expand Down
21 changes: 4 additions & 17 deletions src/main/java/org/apache/lucene/index/IndexGate.java
Original file line number Diff line number Diff line change
@@ -1,25 +1,12 @@
package org.apache.lucene.index;

import java.io.IOException;
import java.io.PrintStream;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Version;
import org.getopt.luke.Luke;
import org.getopt.luke.KeepAllIndexDeletionPolicy;

import java.io.IOException;
import java.util.*;

/**
* This class allows us to peek at various Lucene internals, not available
Expand Down Expand Up @@ -240,7 +227,7 @@ public static boolean preferCompoundFormat(Directory dir) throws Exception {
infos.read(dir);
int compound = 0, nonCompound = 0;
for (int i = 0; i < infos.size(); i++) {
if (((SegmentInfoPerCommit)infos.info(i)).info.getUseCompoundFile()) {
if (infos.info(i).info.getUseCompoundFile()) {
compound++;
} else {
nonCompound++;
Expand Down
25 changes: 14 additions & 11 deletions src/main/java/org/getopt/luke/Luke.java
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ public class Luke extends Thinlet implements ClipboardOwner {

private static final long serialVersionUID = -470469999079073156L;

public static Version LV = Version.LUCENE_43;
public static Version LV = Version.LUCENE_46;

private Directory dir = null;
String pName = null;
Expand All @@ -108,12 +108,14 @@ public class Luke extends Thinlet implements ClipboardOwner {
private Object statmsg = null;
private Object slowstatus = null;
private Object slowmsg = null;
private Analyzer stdAnalyzer = new StandardAnalyzer(LV);
// never read
//private Analyzer stdAnalyzer = new StandardAnalyzer(LV);
//private QueryParser qp = null;
private boolean readOnly = false;
private boolean ram = false;
private boolean keepCommits = false;
private boolean multi = false;
// never read
//private boolean multi = false;
private int tiiDiv = 1;
private IndexCommit currentCommit = null;
private Similarity similarity = null;
Expand Down Expand Up @@ -610,7 +612,8 @@ public void setupInit(Object dialog) {
/**
* Attempt to load the index with parameters specified in the dialog.
* <p>NOTE: this method is invoked from the UI. If you need to open an index
* programmatically, you should use {@link #openIndex(String, boolean, boolean, boolean)} instead.</p>
* programmatically, you should use {@link: openIndex(String, boolean, String, boolean,
boolean, boolean , IndexCommit, int)} instead.</p>
* @param dialog UI dialog with parameters
*/
public void openOk(Object dialog) {
Expand Down Expand Up @@ -818,10 +821,10 @@ public void actionReopen() {

/**
* Open indicated index and re-initialize all GUI and plugins.
* @param pName path to index
* @param name path to index
* @param force if true, and the index is locked, unlock it first. If false, and
* the index is locked, an error will be reported.
* @param readOnly open in read-only mode, and disallow modifications.
* @param ro open in read-only mode, and disallow modifications.
*/
public void openIndex(String name, boolean force, String dirImpl, boolean ro,
boolean ramdir, boolean keepCommits, IndexCommit point, int tiiDivisor) {
Expand Down Expand Up @@ -1384,7 +1387,7 @@ public void showSegments(Object commitsTable) throws Exception {
errorMsg("Error reading segment infos for '" + segName + ": " + e.toString());
return;
}
for (SegmentInfoPerCommit si : infos.asList()) {
for (SegmentCommitInfo si : infos.asList()) {
Object r = create("row");
add(segTable, r);
Object cell = create("cell");
Expand Down Expand Up @@ -1430,7 +1433,7 @@ public void showDiagnostics(Object segmentsTable) {
if (row == null) {
return;
}
SegmentInfoPerCommit si = (SegmentInfoPerCommit)getProperty(row, "si");
SegmentCommitInfo si = (SegmentCommitInfo)getProperty(row, "si");
if (si == null) {
showStatus("Missing SegmentInfoPerCommit???");
return;
Expand Down Expand Up @@ -2316,7 +2319,7 @@ public void abortExport(Object dialog) {

/**
* Optimize the current index
* @param method Thinlet menuitem widget containing the choice of index format.
* Method of optimization is derived from the Thinlet menuitem widget containing the choice of index format.
* If the widget name is "optCompound" then the index will be optimized into compound
* format; otherwise a plain multi-file format will be used.
* <p>NOTE: this method is usually invoked from the GUI, and it also re-initializes GUI
Expand Down Expand Up @@ -4938,7 +4941,7 @@ public void deleteDocs(Object sTable) {
public void actionAbout() {
Object about = addComponent(this, "/xml/about.xml", null, null);
Object lver = find(about, "lver");
setString(lver, "text", "Lucene version: " + Version.LUCENE_45.name());
setString(lver, "text", "Lucene version: " + Version.LUCENE_46.name());
}

/**
Expand Down Expand Up @@ -5183,7 +5186,7 @@ public static Luke startLuke(String[] args) {
Luke luke = new Luke();
DateFormat dateFormat = new SimpleDateFormat("MM-dd-yyyy");
Calendar cal = Calendar.getInstance();
FrameLauncher f = new FrameLauncher("Luke - Lucene Index Toolbox, v 4.5.0 (" + dateFormat.format(cal.getTime()) + ")", luke, 850, 650);
FrameLauncher f = new FrameLauncher("Luke - Lucene Index Toolbox, v 4.6.0 (" + dateFormat.format(cal.getTime()) + ")", luke, 850, 650);
f.setIconImage(Toolkit.getDefaultToolkit().createImage(Luke.class.getResource("/img/luke.gif")));
if (args.length > 0) {
boolean force = false, ro = false, ramdir = false;
Expand Down
28 changes: 13 additions & 15 deletions src/main/java/org/getopt/luke/plugins/AnalyzerToolPlugin.java
Original file line number Diff line number Diff line change
@@ -1,26 +1,19 @@
package org.getopt.luke.plugins;

import java.io.StringReader;
import java.lang.reflect.Constructor;
import java.util.Iterator;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.analysis.tokenattributes.*;
import org.apache.lucene.util.Attribute;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Version;
import org.getopt.luke.LukePlugin;
import org.getopt.luke.Util;

import java.io.StringReader;
import java.lang.reflect.Constructor;
import java.util.Iterator;

public class AnalyzerToolPlugin extends LukePlugin {

/** Default constructor. Initialize analyzers list. */
Expand Down Expand Up @@ -78,7 +71,7 @@ public boolean init() throws Exception {
app.setString(choice, "text", v.toString());
app.putProperty(choice, "version", v);
app.add(aVersion, choice);
if (v.equals(Version.LUCENE_CURRENT)) {
if (v.equals(Version.LUCENE_46)) {
app.setInteger(aVersion, "selected", i);
}
}
Expand Down Expand Up @@ -109,16 +102,21 @@ public void analyze() {
return;
}
}
TokenStream ts = analyzer.tokenStream("text", new StringReader(app


TokenStream ts = analyzer.tokenStream("text", new StringReader(app
.getString(inputText, "text")));
ts.reset();

app.removeAll(resultsList);

while (ts.incrementToken()) {
Object row = app.create("item");
app.setString(row, "text", ((CharTermAttribute)ts.getAttribute(CharTermAttribute.class)).toString());
app.setString(row, "text", (ts.getAttribute(CharTermAttribute.class)).toString());
app.add(resultsList, row);
app.putProperty(row, "state", ts.cloneAttributes());
}
ts.close();
} catch (Throwable t) {
app.showStatus("Error analyzing:" + t.getMessage());
}
Expand Down
42 changes: 26 additions & 16 deletions src/main/java/org/getopt/luke/plugins/FsDirectory.java
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@
public class FsDirectory extends Directory {
private static final Logger LOG = LoggerFactory.getLogger(FsDirectory.class);

Lock lock;
LockFactory lockFactory;

private FileSystem fs;
private Path directory;
private int ioFileBufferSize;
Expand Down Expand Up @@ -81,7 +84,7 @@ public FsDirectory(FileSystem fs, Path directory, boolean create, Configuration
create();
}

if (!fs.getFileStatus(directory).isDir())
if (!fs.getFileStatus(directory).isDirectory())
throw new IOException(directory + " not a directory");
}

Expand All @@ -91,7 +94,7 @@ private void create() throws IOException {
reporter.reportStatus("Created " + directory);
}

if (!fs.getFileStatus(directory).isDir())
if (!fs.getFileStatus(directory).isDirectory())
throw new IOException(directory + " not a directory");

// clear old files
Expand Down Expand Up @@ -169,26 +172,33 @@ public IndexInput openInput(String name, IOContext ctx) throws IOException {
}

public Lock makeLock(final String name) {
return new Lock() {
public boolean obtain() {
return true;
}
public void release() {
}
public boolean isLocked() {
throw new UnsupportedOperationException();
}
public String toString() {
return "Lock@" + new Path(directory, name);
if (lock == null) {
lock = lockFactory.makeLock(name);
}
};
return lock;
}

public synchronized void close() throws IOException {
@Override
public void clearLock(String s) throws IOException {
lock.release();
lock = null;
}

public synchronized void close() throws IOException {
fs.close();
}

public String toString() {
@Override
public void setLockFactory(LockFactory _lockFactory) throws IOException {
lockFactory = _lockFactory;
}

@Override
public LockFactory getLockFactory() {
return lockFactory;
}

public String toString() {
return this.getClass().getName() + "@" + directory;
}

Expand Down
15 changes: 7 additions & 8 deletions src/main/java/org/getopt/luke/plugins/HadoopPlugin.java
Original file line number Diff line number Diff line change
@@ -1,11 +1,5 @@
package org.getopt.luke.plugins;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.TreeMap;
import java.util.Map.Entry;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
Expand All @@ -14,10 +8,15 @@
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.MultiReader;
import org.getopt.luke.IntPair;
import org.getopt.luke.LukePlugin;
import org.getopt.luke.SlowThread;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map.Entry;
import java.util.TreeMap;

public class HadoopPlugin extends LukePlugin {

private String lastMsg = "?";
Expand Down Expand Up @@ -123,7 +122,7 @@ private void openIndex(String uriTxt) {
FileStatus[] stats = fs.listStatus(path);
boolean hasParts = true;
for (FileStatus s : stats) {
if (s.isDir() && s.getPath().getName().startsWith("part-")) {
if (s.isDirectory() && s.getPath().getName().startsWith("part-")) {
continue;
} else {
hasParts = false;
Expand Down
4 changes: 2 additions & 2 deletions src/main/resources/xml/about.xml
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
<?xml version="1.0" encoding="UTF-8"?>
<dialog modal="true" name="about" columns="1" gap="8" icon="/img/luke.gif" text=" About" left="4" right="4" bottom="4" top="4" close="remove(about)">
<label halign="center" valign="center" text="v 4.5.0 (2013-11-29)" font="10" icon="/img/luke-big.gif"/>
<label halign="center" valign="center" text="v 4.6.0 (2013-12-05)" font="10" icon="/img/luke-big.gif"/>
<button type="link" tooltip="Go to Luke home page" action="goUrl(this)" halign="center" property="url=http://www.getopt.org/luke" foreground="#5aaa88" text="Lucene Index Toolbox" font="18 bold"/>
<label halign="center" font="13" text="Created by Andrzej Bialecki &lt;[email protected]&gt;"/>
<label halign="center" font="13" text="Maintained and further developed by Dmitry Kan &lt;[email protected]&gt;"/>
<label halign="center" font="italic 11" text="GUI client of the best Java search library"/>
<button type="link" icon="/img/lucene.gif" tooltip="Go to Lucene home page" action="goUrl(this)" halign="center" property="url=http://lucene.apache.org/" foreground="#5aaa88" text="Apache Lucene - home page"/>
<label halign="center" font="11" name="lver" text="4.5.0"/>
<label halign="center" font="11" name="lver" text=""/>
<textarea columns="55" rows="3" editable="false" font="10" border="false" wrap="true" text="Luke is distributed under Apache Software License. Please see http://www.apache.org/LICENSE-2.0 for details.&#10;This software includes Thinlet library (http://thinlet.sourceforge.net)"/>
</dialog>

0 comments on commit 87e7d66

Please sign in to comment.