Add TestSumLogs

Author: jdkato

extract.go

@@ -14,6 +14,53 @@ import (
 	"gonum.org/v1/gonum/mat"
 )
 
+var maxLogDiff = math.Log2(1e-30)
+
+type mappedProbDist struct {
+	dict map[string]float64
+	log  bool
+}
+
+func (m *mappedProbDist) prob(label string) float64 {
+	if p, found := m.dict[label]; found {
+		return math.Pow(2, p)
+	}
+	return 0.0
+}
+
+func (m *mappedProbDist) max() string {
+	var class string
+	max := math.Inf(-1)
+	for label, value := range m.dict {
+		if value > max {
+			max = value
+			class = label
+		}
+	}
+	return class
+}
+
+func newMappedProbDist(dict map[string]float64, normalize bool) *mappedProbDist {
+	if normalize {
+		values := []float64{}
+		for _, v := range dict {
+			values = append(values, v)
+		}
+		sum := sumLogs(values)
+		if sum <= math.Inf(-1) {
+			p := math.Log2(1.0 / float64(len(dict)))
+			for k := range dict {
+				dict[k] = p
+			}
+		} else {
+			for k := range dict {
+				dict[k] -= sum
+			}
+		}
+	}
+	return &mappedProbDist{dict: dict, log: true}
+}
+
 type encodedValue struct {
 	key   int
 	value int
@@ -480,3 +527,24 @@ func empiricalCount(corpus featureSet, encoding *binaryMaxentClassifier) *mat.Ve
 	}
 	return count
 }
+
+func addLogs(x, y float64) float64 {
+	if x < y+maxLogDiff {
+		return y
+	} else if y < x+maxLogDiff {
+		return x
+	}
+	base := math.Min(x, y)
+	return base + math.Log2(math.Pow(2, x-base)+math.Pow(2, y-base))
+}
+
+func sumLogs(logs []float64) float64 {
+	if len(logs) == 0 {
+		return math.Inf(-1)
+	}
+	sum := logs[0]
+	for _, log := range logs[1:] {
+		sum = addLogs(sum, log)
+	}
+	return sum
+}

extract_test.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"io"
 	"io/ioutil"
+	"math"
 	"path/filepath"
 	"reflect"
 	"testing"
@@ -59,6 +60,10 @@ func split(data []prodigyOuput) ([]EntityContext, []prodigyOuput) {
 	return train, test
 }
 
+func TestSumLogs(t *testing.T) {
+	assert.Equal(t, 3.0, sumLogs([]float64{math.Log2(3), math.Log2(5)}))
+}
+
 func TestNERProdigy(t *testing.T) {
 	data := filepath.Join(testdata, "reddit_product.jsonl")
 

utilities.go

@@ -3,81 +3,12 @@ package prose
 import (
 	"bytes"
 	"encoding/gob"
-	"math"
 	"os"
 	"path"
 	"strconv"
 	"strings"
 )
 
-var maxLogDiff = math.Log2(1e-30)
-
-type mappedProbDist struct {
-	dict map[string]float64
-	log  bool
-}
-
-func (m *mappedProbDist) prob(label string) float64 {
-	if p, found := m.dict[label]; found {
-		return math.Pow(2, p)
-	}
-	return 0.0
-}
-
-func (m *mappedProbDist) max() string {
-	var class string
-	max := math.Inf(-1)
-	for label, value := range m.dict {
-		if value > max {
-			max = value
-			class = label
-		}
-	}
-	return class
-}
-
-func newMappedProbDist(dict map[string]float64, normalize bool) *mappedProbDist {
-	if normalize {
-		values := []float64{}
-		for _, v := range dict {
-			values = append(values, v)
-		}
-		sum := sumLogs(values)
-		if sum <= math.Inf(-1) {
-			p := math.Log2(1.0 / float64(len(dict)))
-			for k := range dict {
-				dict[k] = p
-			}
-		} else {
-			for k := range dict {
-				dict[k] -= sum
-			}
-		}
-	}
-	return &mappedProbDist{dict: dict, log: true}
-}
-
-func addLogs(x, y float64) float64 {
-	if x < y+maxLogDiff {
-		return y
-	} else if y < x+maxLogDiff {
-		return x
-	}
-	base := math.Min(x, y)
-	return base + math.Log2(math.Pow(2, x-base)+math.Pow(2, y-base))
-}
-
-func sumLogs(logs []float64) float64 {
-	if len(logs) == 0 {
-		return math.Inf(-1)
-	}
-	sum := logs[0]
-	for _, log := range logs[1:] {
-		sum = addLogs(sum, log)
-	}
-	return sum
-}
-
 // checkError panics if `err` is not `nil`.
 func checkError(err error) {
 	if err != nil {