-
Notifications
You must be signed in to change notification settings - Fork 1
/
structer.go
85 lines (74 loc) · 1.57 KB
/
structer.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
package siw
import (
"log"
"net/http"
"os"
"time"
)
type ClerkLog struct {
*log.Logger
}
func newClerkLog(prefix string) *ClerkLog {
var f, err = os.Create("./log/simple_words.log")
if err != nil {
log.Fatal(err)
}
return &ClerkLog{
log.New(f, prefix, (log.Ldate | log.Ltime | log.Lmicroseconds | log.Lshortfile)),
}
}
var clog = newClerkLog("siw: ")
// Highest level container for web content
type Collection struct {
DocList []*Document
BuildTime time.Duration
}
// Core data container for web content,
// including parsed content such as words and sentences
type Document struct {
id int
label string
words []string
sentences []string
dBuildTime time.Duration
httpres *HttpResponse
}
// Data for errors with Transporter
type transportError struct {
Error error
Message string
Url string
Code int
}
// Holds data about errors in AsyncWeb,
// including the request that was made
type asyncError struct {
Error error
Message string
Url string
Code int
errRequest *http.Request
}
// Holds web responses
// Also contains HttpRequest and asyncError
// for easier access to all data in the Document
type HttpResponse struct {
url string
response *http.Response
request *HttpRequest
err error
asyncErr *asyncError
}
// Holds web requests
type HttpRequest struct {
url string
request *http.Request
err error
}
// Basic Data for indexer, used for kicking off a run
type Indexer struct {
readTimeout time.Duration
reqTimeout time.Duration
reqMethod string
uris []string
}