Merge branch 'v2' of ssh://github.com/jdkato/prose into v2

Author: jdkato

extract.go

@@ -209,7 +209,7 @@ func extractFeatures(tokens []*Token, history []string) []feature {
 	return features
 }
 
-func assignLabels(tokens []*Token, entity EntityContext) []string {
+func assignLabels(tokens []*Token, entity *EntityContext) []string {
 	history := []string{}
 	for range tokens {
 		history = append(history, "O")
@@ -236,7 +236,8 @@ func assignLabels(tokens []*Token, entity EntityContext) []string {
 func makeCorpus(data []EntityContext, tagger *perceptronTagger) featureSet {
 	tokenizer := newIterTokenizer()
 	corpus := featureSet{}
-	for _, entry := range data {
+	for i := range data {
+		entry := &data[i]
 		tokens := tagger.tag(tokenizer.tokenize(entry.Text))
 		history := assignLabels(tokens, entry)
 		for _, element := range extractFeatures(tokens, history) {

extract_test.go

@@ -45,14 +45,14 @@ func split(data []prodigyOuput) ([]EntityContext, []prodigyOuput) {
 	cutoff := int(float64(len(data)) * 0.8)
 
 	train, test := []EntityContext{}, []prodigyOuput{}
-	for i, entry := range data {
+	for i := range data {
 		if i < cutoff {
 			train = append(train, EntityContext{
-				Text:   entry.Text,
-				Spans:  entry.Spans,
-				Accept: entry.Answer == "accept"})
+				Text:   data[i].Text,
+				Spans:  data[i].Spans,
+				Accept: data[i].Answer == "accept"})
 		} else {
-			test = append(test, entry)
+			test = append(test, data[i])
 		}
 	}
 

segment_test.go

@@ -37,8 +37,8 @@ func BenchmarkPunkt(b *testing.B) {
 
 	checkError(json.Unmarshal(cases, &tests))
 	for n := 0; n < b.N; n++ {
-		for _, test := range tests {
-			makeSegmenter(test.Input)
+		for i := range tests {
+			makeSegmenter(tests[i].Input)
 		}
 	}
 }

tokenize_test.go

@@ -20,8 +20,9 @@ func makeDoc(text string) (*Document, error) {
 
 func getTokenText(doc *Document) []string {
 	observed := []string{}
-	for _, tok := range doc.Tokens() {
-		observed = append(observed, tok.Text)
+	tokens := doc.Tokens()
+	for i := range tokens {
+		observed = append(observed, tokens[i].Text)
 	}
 	return observed
 }