Skip to content

Commit

Permalink
update api/fetch log
Browse files Browse the repository at this point in the history
  • Loading branch information
@Miachol
Miachol committed Jan 17, 2020
1 parent 1053573 commit 1c1cf97
Show file tree
Hide file tree
Showing 7 changed files with 88 additions and 95 deletions.
31 changes: 16 additions & 15 deletions api/fetch/biotools.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,22 +10,23 @@ import (
const BioToolsHost = "https://bio.tools/api/tool/"

// BioTools access https://clinicaltrials.gov API
func BioTools(endpoints *types.BioToolsEndpoints, bapiClis *types.BapiClisT, f func()) bool {
if bapiClis.Format == "" {
bapiClis.Format = "json"
func BioTools(endpoints *types.BioToolsEndpoints, BapiClis *types.BapiClisT, f func()) bool {
setLog(BapiClis)
if BapiClis.Format == "" {
BapiClis.Format = "json"
}
netopt := setNetOpt(bapiClis)
url := BioToolsHost + setBioToolsQuerySuffix(endpoints, bapiClis)
netopt := setNetOpt(BapiClis)
url := BioToolsHost + setBioToolsQuerySuffix(endpoints, BapiClis)
if url == BioToolsHost || url == BioToolsHost+"?format=json" {
return false
}
f()
queryAPI("bio.tools", url, bapiClis, netopt)
queryAPI("bio.tools", url, BapiClis, netopt)

return true
}

func setBioToolsQuerySuffix(endpoints *types.BioToolsEndpoints, bapiClis *types.BapiClisT) (suffix string) {
func setBioToolsQuerySuffix(endpoints *types.BioToolsEndpoints, BapiClis *types.BapiClisT) (suffix string) {
suffixList := []string{}
if endpoints.Tool != "" {
suffix = suffix + endpoints.Tool + "/"
Expand All @@ -51,17 +52,17 @@ func setBioToolsQuerySuffix(endpoints *types.BioToolsEndpoints, bapiClis *types.
if endpoints.Publication != "" {
suffixList = append(suffixList, `publication=`+endpoints.Publication)
}
if bapiClis.Query != "" {
suffixList = append(suffixList, "q="+bapiClis.Query)
if BapiClis.Query != "" {
suffixList = append(suffixList, "q="+BapiClis.Query)
}
if bapiClis.Format != "" {
suffixList = append(suffixList, "format="+bapiClis.Format)
if BapiClis.Format != "" {
suffixList = append(suffixList, "format="+BapiClis.Format)
}
if bapiClis.Size != -1 {
suffixList = append(suffixList, "page="+strconv.Itoa(bapiClis.From+bapiClis.Size))
if BapiClis.Size != -1 {
suffixList = append(suffixList, "page="+strconv.Itoa(BapiClis.From+BapiClis.Size))
}
if bapiClis.Extra != "" {
suffixList = append(suffixList, bapiClis.Extra)
if BapiClis.Extra != "" {
suffixList = append(suffixList, BapiClis.Extra)
}
if len(suffixList) > 0 {
suffix = suffix + "?" + strings.Join(suffixList, "&")
Expand Down
39 changes: 20 additions & 19 deletions api/fetch/cligov.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,22 +10,23 @@ import (
const CligovHost = "https://clinicaltrials.gov/api/"

// Cligov access https://clinicaltrials.gov API
func Cligov(endpoints *types.CligovEndpoints, bapiClis *types.BapiClisT, f func()) bool {
if bapiClis.Format == "" {
bapiClis.Format = "json"
func Cligov(endpoints *types.CligovEndpoints, BapiClis *types.BapiClisT, f func()) bool {
setLog(BapiClis)
if BapiClis.Format == "" {
BapiClis.Format = "json"
}
netopt := setNetOpt(bapiClis)
url := CligovHost + setCligovQuerySuffix(endpoints, bapiClis)
netopt := setNetOpt(BapiClis)
url := CligovHost + setCligovQuerySuffix(endpoints, BapiClis)
if url == CligovHost || url == CligovHost+"?fmt=json" {
return false
}
f()
queryAPI("clinicaltrials.gov", url, bapiClis, netopt)
queryAPI("clinicaltrials.gov", url, BapiClis, netopt)

return true
}

func setCligovQuerySuffix(endpoints *types.CligovEndpoints, bapiClis *types.BapiClisT) (suffix string) {
func setCligovQuerySuffix(endpoints *types.CligovEndpoints, BapiClis *types.BapiClisT) (suffix string) {
suffixList := []string{}
if endpoints.InfoDataVrs {
suffix = "info/data_vrs"
Expand All @@ -48,28 +49,28 @@ func setCligovQuerySuffix(endpoints *types.CligovEndpoints, bapiClis *types.Bapi
} else if endpoints.FieldValues {
suffix = "query/field_values"
}
if bapiClis.Query != "" {
suffixList = append(suffixList, "expr="+bapiClis.Query)
if BapiClis.Query != "" {
suffixList = append(suffixList, "expr="+BapiClis.Query)
}
if endpoints.Field != "" {
suffixList = append(suffixList, "field="+endpoints.Field)
}
if endpoints.Fields != "" {
suffixList = append(suffixList, "fields="+endpoints.Fields)
}
if bapiClis.Format != "" {
suffixList = append(suffixList, "fmt="+bapiClis.Format)
if BapiClis.Format != "" {
suffixList = append(suffixList, "fmt="+BapiClis.Format)
}
if bapiClis.From != -1 {
suffixList = append(suffixList, "min_rnk="+strconv.Itoa(bapiClis.From))
if BapiClis.From != -1 {
suffixList = append(suffixList, "min_rnk="+strconv.Itoa(BapiClis.From))
}
if bapiClis.Size != -1 && bapiClis.From != -1 {
suffixList = append(suffixList, "max_rnk="+strconv.Itoa(bapiClis.From+bapiClis.Size))
} else if bapiClis.Size != -1 {
suffixList = append(suffixList, "max_rnk="+strconv.Itoa(1+bapiClis.Size))
if BapiClis.Size != -1 && BapiClis.From != -1 {
suffixList = append(suffixList, "max_rnk="+strconv.Itoa(BapiClis.From+BapiClis.Size))
} else if BapiClis.Size != -1 {
suffixList = append(suffixList, "max_rnk="+strconv.Itoa(1+BapiClis.Size))
}
if bapiClis.Extra != "" {
suffixList = append(suffixList, bapiClis.Extra)
if BapiClis.Extra != "" {
suffixList = append(suffixList, BapiClis.Extra)
}
if len(suffixList) > 0 {
suffix = suffix + "?" + strings.Join(suffixList, "&")
Expand Down
15 changes: 8 additions & 7 deletions api/fetch/dataset2tools.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,15 @@ import (
const Dataset2toolsHost = "http://amp.pharm.mssm.edu/datasets2tools/api/search?"

// Dataset2tools access http://amp.pharm.mssm.edu/datasets2tools/ API
func Dataset2tools(endpoints *types.Datasets2toolsEndpoints, bapiClis *types.BapiClisT) {
netopt := setNetOpt(bapiClis)
url := Dataset2toolsHost + setDatasets2toolsQuerySuffix(endpoints, bapiClis)
queryAPI("datasets2tools", url, bapiClis, netopt)
func Dataset2tools(endpoints *types.Datasets2toolsEndpoints, BapiClis *types.BapiClisT) {
setLog(BapiClis)
netopt := setNetOpt(BapiClis)
url := Dataset2toolsHost + setDatasets2toolsQuerySuffix(endpoints, BapiClis)
queryAPI("datasets2tools", url, BapiClis, netopt)
return
}

func setDatasets2toolsQuerySuffix(endpoints *types.Datasets2toolsEndpoints, bapiClis *types.BapiClisT) (suffix string) {
func setDatasets2toolsQuerySuffix(endpoints *types.Datasets2toolsEndpoints, BapiClis *types.BapiClisT) (suffix string) {
suffixList := []string{}
if endpoints.ObjectType != "" {
suffixList = append(suffixList, "object_type="+endpoints.ObjectType)
Expand All @@ -43,8 +44,8 @@ func setDatasets2toolsQuerySuffix(endpoints *types.Datasets2toolsEndpoints, bapi
if endpoints.PageSize != -1 {
suffixList = append(suffixList, "page_size="+strconv.Itoa(endpoints.PageSize))
}
if bapiClis.Extra != "" {
suffixList = append(suffixList, bapiClis.Extra)
if BapiClis.Extra != "" {
suffixList = append(suffixList, BapiClis.Extra)
}
if len(suffixList) > 0 {
suffix = strings.Join(suffixList, "&")
Expand Down
21 changes: 11 additions & 10 deletions api/fetch/gdc.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,9 @@ var gdcapis = []string{"status", "projects", "cases", "files", "annotations",
var pg *mpb.Progress

// Gdc accesss https://api.gdc.cancer.gov data
func Gdc(endpoint *types.GdcEndpoints, bapiClis *types.BapiClisT) {
netopt := setNetOpt(bapiClis)
func Gdc(endpoint *types.GdcEndpoints, BapiClis *types.BapiClisT) {
setLog(BapiClis)
netopt := setNetOpt(BapiClis)
client := cnet.NewHTTPClient(netopt.Timeout, netopt.Proxy)
host := GdcAPIHost
if endpoint.Legacy {
Expand All @@ -48,9 +49,9 @@ func Gdc(endpoint *types.GdcEndpoints, bapiClis *types.BapiClisT) {
queryFlag = ""
f := v.Field(i)
if f.Kind() == reflect.String && f.String() != "" {
req, queryFlag = setGdcReq(host, i, endpoint, bapiClis)
req, queryFlag = setGdcReq(host, i, endpoint, BapiClis)
} else if f.Kind() == reflect.Bool && f.Bool() {
req, queryFlag = setGdcReq(host, i, endpoint, bapiClis)
req, queryFlag = setGdcReq(host, i, endpoint, BapiClis)
}
if queryFlag == "" {
continue
Expand All @@ -66,7 +67,7 @@ func Gdc(endpoint *types.GdcEndpoints, bapiClis *types.BapiClisT) {
} else {
return
}
outfn := cnet.ParseOutfnFromHeader(bapiClis.Outfn, resp, endpoint.ExtraParams.RemoteName)
outfn := cnet.ParseOutfnFromHeader(BapiClis.Outfn, resp, endpoint.ExtraParams.RemoteName)

defer resp.Body.Close()
of := cio.NewOutStream(outfn, req.URL.String())
Expand All @@ -89,9 +90,9 @@ func Gdc(endpoint *types.GdcEndpoints, bapiClis *types.BapiClisT) {
}
}

func setGdcReq(host string, i int, endpoint *types.GdcEndpoints, bapiClis *types.BapiClisT) (*http.Request, string) {
func setGdcReq(host string, i int, endpoint *types.GdcEndpoints, BapiClis *types.BapiClisT) (*http.Request, string) {
queryFlag := gdcapis[i]
suffix := setGdcQuerySuffix(queryFlag, endpoint, bapiClis)
suffix := setGdcQuerySuffix(queryFlag, endpoint, BapiClis)
method := "GET"
req, err := http.NewRequest(method, host+"/"+gdcapis[i]+suffix, nil)
cnet.SetDefaultReqHeader(req)
Expand All @@ -101,7 +102,7 @@ func setGdcReq(host string, i int, endpoint *types.GdcEndpoints, bapiClis *types
return req, queryFlag
}

func setGdcQuerySuffix(queryFlag string, endpoint *types.GdcEndpoints, bapiClis *types.BapiClisT) (suffix string) {
func setGdcQuerySuffix(queryFlag string, endpoint *types.GdcEndpoints, BapiClis *types.BapiClisT) (suffix string) {
queryStr := ""
suffixList := []string{}
if endpoint.ExtraParams.Query != "" {
Expand Down Expand Up @@ -148,8 +149,8 @@ func setGdcQuerySuffix(queryFlag string, endpoint *types.GdcEndpoints, bapiClis
if endpoint.ExtraParams.Pretty {
suffixList = append(suffixList, "pretty=true")
}
if bapiClis.Extra != "" {
suffixList = append(suffixList, bapiClis.Extra)
if BapiClis.Extra != "" {
suffixList = append(suffixList, BapiClis.Extra)
}
if len(suffixList) > 0 && queryStr != "" {
if !strings.Contains(queryStr, "?") {
Expand Down
28 changes: 0 additions & 28 deletions api/fetch/global_var.go
View file

This file was deleted.

29 changes: 14 additions & 15 deletions api/fetch/ncbi.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package fetch

import (
"bytes"
"fmt"
"io"
"os"
"time"
Expand All @@ -15,48 +14,48 @@ import (
)

// Ncbi modified from https://github.com/biogo/ncbi BSD license
func Ncbi(bapiClis *types.BapiClisT, ncbiClis *types.NcbiClisT) {
SetLogStream(bapiClis.Verbose == 0, bapiClis.SaveLog, fmt.Sprintf("%s/%s.log", bapiClis.LogDir, bapiClis.TaskID))
ncbi.SetTimeout(time.Duration(bapiClis.Timeout) * time.Second)
func Ncbi(BapiClis *types.BapiClisT, ncbiClis *types.NcbiClisT) {
setLog(BapiClis)
ncbi.SetTimeout(time.Duration(BapiClis.Timeout) * time.Second)
tool := "entrez.example"
h := entrez.History{}
parms := entrez.Parameters{
APIKey: "193124979d2e7f360c150dadc5b1e3bfec09",
}
s, err := entrez.DoSearch(ncbiClis.NcbiDB, bapiClis.Query, &parms, &h, tool, bapiClis.Email)
s, err := entrez.DoSearch(ncbiClis.NcbiDB, BapiClis.Query, &parms, &h, tool, BapiClis.Email)
if err != nil {
log.Fatalf("error: %v\n", err)
}
log.Infof("Available retrieve %d records.", s.Count)
from, end := cnet.SetQueryFromEnd(bapiClis.From, bapiClis.Size, s.Count)
from, end := cnet.SetQueryFromEnd(BapiClis.From, BapiClis.Size, s.Count)
log.Infof("Will retrieve %d records, from %d to %d.", end-from, from+1, end)

of := cio.NewOutStream(bapiClis.Outfn, "")
of := cio.NewOutStream(BapiClis.Outfn, "")
defer of.Close()
var (
buf = &bytes.Buffer{}
p = &entrez.Parameters{RetMax: ncbiClis.NcbiRetmax, RetType: bapiClis.Format, RetMode: "text"}
p = &entrez.Parameters{RetMax: ncbiClis.NcbiRetmax, RetType: BapiClis.Format, RetMode: "text"}
bn, n int64
)
if p.RetMax > end-from {
p.RetMax = end - from
}
for p.RetStart = from; p.RetStart < end; p.RetStart += p.RetMax {
log.Infof("Attempting to retrieve %d records: %d-%d with %d retries.", p.RetMax, p.RetStart+1, p.RetMax+p.RetStart, bapiClis.Retries)
log.Infof("Attempting to retrieve %d records: %d-%d with %d retries.", p.RetMax, p.RetStart+1, p.RetMax+p.RetStart, BapiClis.Retries)
var t int
for t = 0; t < bapiClis.Retries; t++ {
for t = 0; t < BapiClis.Retries; t++ {
buf.Reset()
var (
r io.ReadCloser
_bn int64
)
r, err = entrez.Fetch(ncbiClis.NcbiDB, p, tool, bapiClis.Email, &h)
r, err = entrez.Fetch(ncbiClis.NcbiDB, p, tool, BapiClis.Email, &h)
if err != nil {
if r != nil {
r.Close()
}
log.Warnf("Failed to retrieve on attempt %d... error: %v ... retrying after %d seconds.", t+1, err, bapiClis.RetSleepTime)
time.Sleep(time.Duration(bapiClis.RetSleepTime) * time.Second)
log.Warnf("Failed to retrieve on attempt %d... error: %v ... retrying after %d seconds.", t+1, err, BapiClis.RetSleepTime)
time.Sleep(time.Duration(BapiClis.RetSleepTime) * time.Second)
continue
}
_bn, err = io.Copy(buf, r)
Expand All @@ -65,8 +64,8 @@ func Ncbi(bapiClis *types.BapiClisT, ncbiClis *types.NcbiClisT) {
bn += _bn
break
}
log.Warnf("Failed to buffer on attempt %d... error: %v ... retrying after %d seconds.", t+1, err, bapiClis.RetSleepTime)
time.Sleep(time.Duration(bapiClis.RetSleepTime) * time.Second)
log.Warnf("Failed to buffer on attempt %d... error: %v ... retrying after %d seconds.", t+1, err, BapiClis.RetSleepTime)
time.Sleep(time.Duration(BapiClis.RetSleepTime) * time.Second)
}
if err != nil {
os.Exit(1)
Expand Down
20 changes: 19 additions & 1 deletion api/fetch/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,20 @@ import (
"io"
"io/ioutil"
"net/http"
"os"
"path/filepath"

"github.com/openbiox/bget/api/types"
cio "github.com/openbiox/ligo/io"
clog "github.com/openbiox/ligo/log"
cnet "github.com/openbiox/ligo/net"
"github.com/tidwall/pretty"
)

var log = clog.Logger
var logPrefix string
var logCon io.Writer

func setNetOpt(bapiClis *types.BapiClisT) *cnet.Params {
var netopt = &cnet.Params{
Pbar: pg,
Expand All @@ -22,7 +29,6 @@ func setNetOpt(bapiClis *types.BapiClisT) *cnet.Params {
netopt.Timeout = bapiClis.Timeout
netopt.RetSleepTime = bapiClis.RetSleepTime
netopt.Proxy = bapiClis.Proxy
SetLogStream(bapiClis.Verbose == 0, bapiClis.SaveLog, fmt.Sprintf("%s/%s.log", bapiClis.LogDir, bapiClis.TaskID))
return netopt
}

Expand Down Expand Up @@ -66,3 +72,15 @@ func queryAPI(siteName, url string, bapiClis *types.BapiClisT, netopt *cnet.Para

return
}

func setLog(BapiClis *types.BapiClisT) {
if BapiClis.SaveLog {
if BapiClis.LogDir == "" {
BapiClis.LogDir = filepath.Join(os.TempDir(), "_log")
}
logPrefix = fmt.Sprintf("%s/%s", BapiClis.LogDir, BapiClis.TaskID)
cio.CreateDir(BapiClis.LogDir)
logCon, _ = cio.Open(logPrefix + ".log")
}
clog.SetLogStream(log, BapiClis.Verbose == 0, BapiClis.SaveLog, &logCon)
}

0 comments on commit 1c1cf97

Please sign in to comment.