Refactoring collector

csv-collector/collector/base.go

@@ -0,0 +1,39 @@
+// Copyright (c) 2020, Marcelo Jorge Vieira
+// Licensed under the AGPL-3.0+ License
+
+package collector
+
+import (
+	"github.com/gosimple/slug"
+	"github.com/olhoneles/politicos-go/db"
+	"github.com/olhoneles/politicos-go/politicos"
+	log "github.com/sirupsen/logrus"
+)
+
+func collectorBase(q politicos.Queryable, opts db.UniqueOptions) error {
+	log.Debug("[Collector] CollectorBase")
+
+	dbInstance, err := db.NewMongoSession()
+	if err != nil {
+		return err
+	}
+
+	results, err := dbInstance.GetUnique(&politicos.Candidatures{}, q, opts)
+	if err != nil {
+		return err
+	}
+
+	// FIXME
+	items := []interface{}{}
+	for _, p := range results {
+		p.SetSlug(slug.Make(p.GetID()))
+		items = append(items, p)
+	}
+
+	_, err = dbInstance.InsertMany(q, items)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

csv-collector/collector/candidacy_status.go

@@ -4,7 +4,6 @@
 package collector
 
 import (
-	"github.com/gosimple/slug"
 	"github.com/olhoneles/politicos-go/db"
 	"github.com/olhoneles/politicos-go/politicos"
 	log "github.com/sirupsen/logrus"
@@ -15,37 +14,14 @@ import (
 func ProcessAllCandidaciesStatus() error {
 	log.Debug("[Collector] ProcessAllCandidaciesStatus")
 
-	dbInstance, err := db.NewMongoSession()
-	if err != nil {
-		return err
-	}
-
 	opts := db.UniqueOptions{
 		IDs: bson.D{
 			primitive.E{Key: "tseId", Value: "$cd_sit_tot_turno"},
 			primitive.E{Key: "name", Value: "$ds_sit_tot_turno"},
 		},
 	}
 
-	results, err := dbInstance.GetUnique(
-		&politicos.Candidatures{},
-		&politicos.CandidacyStatus{},
-		opts,
-	)
-	if err != nil {
-		return err
-	}
-
-	// FIXME
-	candidaciesStatus := []interface{}{}
-	for _, p := range results {
-		c := p.(*politicos.CandidacyStatus)
-		c.Slug = slug.Make(c.Name)
-		candidaciesStatus = append(candidaciesStatus, c)
-	}
-
-	_, err = dbInstance.InsertMany(&politicos.CandidacyStatus{}, candidaciesStatus)
-	if err != nil {
+	if err := collectorBase(&politicos.CandidacyStatus{}, opts); err != nil {
 		return err
 	}
 

csv-collector/collector/education.go

@@ -4,7 +4,6 @@
 package collector
 
 import (
-	"github.com/gosimple/slug"
 	"github.com/olhoneles/politicos-go/db"
 	"github.com/olhoneles/politicos-go/politicos"
 	log "github.com/sirupsen/logrus"
@@ -15,37 +14,14 @@ import (
 func ProcessAllEducations() error {
 	log.Debug("[Collector] ProcessAllEducations")
 
-	dbInstance, err := db.NewMongoSession()
-	if err != nil {
-		return err
-	}
-
 	opts := db.UniqueOptions{
 		IDs: bson.D{
 			primitive.E{Key: "tseId", Value: "$cd_grau_instrucao"},
 			primitive.E{Key: "name", Value: "$ds_grau_instrucao"},
 		},
 	}
 
-	results, err := dbInstance.GetUnique(
-		&politicos.Candidatures{},
-		&politicos.Education{},
-		opts,
-	)
-	if err != nil {
-		return err
-	}
-
-	// FIXME
-	educations := []interface{}{}
-	for _, p := range results {
-		e := p.(*politicos.Education)
-		e.Slug = slug.Make(e.Name)
-		educations = append(educations, e)
-	}
-
-	_, err = dbInstance.InsertMany(&politicos.Education{}, educations)
-	if err != nil {
+	if err := collectorBase(&politicos.Education{}, opts); err != nil {
 		return err
 	}
 

csv-collector/collector/political_office.go

@@ -14,35 +14,14 @@ import (
 func ProcessAllPoliticalOffices() error {
 	log.Debug("[Collector] ProcessAllPoliticalOffices")
 
-	dbInstance, err := db.NewMongoSession()
-	if err != nil {
-		return err
-	}
-
 	opts := db.UniqueOptions{
 		IDs: bson.D{
 			primitive.E{Key: "tseId", Value: "$cd_cargo"},
 			primitive.E{Key: "name", Value: "$ds_cargo"},
 		},
 	}
 
-	results, err := dbInstance.GetUnique(
-		&politicos.Candidatures{},
-		&politicos.PoliticalOffice{},
-		opts,
-	)
-	if err != nil {
-		return err
-	}
-
-	// FIXME
-	politicalOffices := []interface{}{}
-	for _, p := range results {
-		politicalOffices = append(politicalOffices, p)
-	}
-
-	_, err = dbInstance.InsertMany(&politicos.PoliticalOffice{}, politicalOffices)
-	if err != nil {
+	if err := collectorBase(&politicos.PoliticalOffice{}, opts); err != nil {
 		return err
 	}
 

csv-collector/collector/political_party.go

@@ -14,11 +14,6 @@ import (
 func ProcessAllPoliticalParties() error {
 	log.Debug("[Collector] ProcessAllPoliticalParties")
 
-	dbInstance, err := db.NewMongoSession()
-	if err != nil {
-		return err
-	}
-
 	opts := db.UniqueOptions{
 		IDs: bson.D{
 			primitive.E{Key: "siglum", Value: "$sg_partido"},
@@ -27,23 +22,7 @@ func ProcessAllPoliticalParties() error {
 		},
 	}
 
-	results, err := dbInstance.GetUnique(
-		&politicos.Candidatures{},
-		&politicos.PoliticalParty{},
-		opts,
-	)
-	if err != nil {
-		return err
-	}
-
-	// FIXME
-	politicalParties := []interface{}{}
-	for _, p := range results {
-		politicalParties = append(politicalParties, p)
-	}
-
-	_, err = dbInstance.InsertMany(&politicos.PoliticalParty{}, politicalParties)
-	if err != nil {
+	if err := collectorBase(&politicos.PoliticalParty{}, opts); err != nil {
 		return err
 	}
 

politicos/candidacy_status.go

@@ -16,3 +16,11 @@ func (c CandidacyStatus) GetCollectionName() string {
 func (c CandidacyStatus) Cast() Queryable {
 	return &c
 }
+
+func (c CandidacyStatus) GetID() string {
+	return c.Name
+}
+
+func (c *CandidacyStatus) SetSlug(slug string) {
+	c.Slug = slug
+}

politicos/candidatures.go

@@ -5,6 +5,7 @@ package politicos
 
 // FIXME
 type Candidatures struct {
+	Slug                          string `json:"slug" bson:"slug"`
 	DT_GERACAO                    string
 	HH_GERACAO                    string
 	ANO_ELEICAO                   string
@@ -77,3 +78,11 @@ func (c Candidatures) GetCollectionName() string {
 func (c Candidatures) Cast() Queryable {
 	return &c
 }
+
+func (c Candidatures) GetID() string {
+	return c.SQ_CANDIDATO
+}
+
+func (c *Candidatures) SetSlug(slug string) {
+	c.Slug = slug
+}

politicos/education.go

@@ -16,3 +16,11 @@ func (c Education) GetCollectionName() string {
 func (c Education) Cast() Queryable {
 	return &c
 }
+
+func (c Education) GetID() string {
+	return c.Name
+}
+
+func (c *Education) SetSlug(slug string) {
+	c.Slug = slug
+}

politicos/political_office.go

@@ -19,3 +19,11 @@ func (c PoliticalOffice) GetCollectionName() string {
 func (c PoliticalOffice) Cast() Queryable {
 	return &c
 }
+
+func (c PoliticalOffice) GetID() string {
+	return c.Name
+}
+
+func (c *PoliticalOffice) SetSlug(slug string) {
+	c.Slug = slug
+}

politicos/politician.go

@@ -4,12 +4,12 @@
 package politicos
 
 type Politician struct {
-	CPF                           string
-	Data                          []Candidatures `bson:"data"`
 	CD_COR_RACA                   string
 	CD_ESTADO_CIVIL               string
 	CD_MUNICIPIO_NASCIMENTO       string
 	CD_NACIONALIDADE              string
+	CPF                           string
+	Data                          []Candidatures `bson:"data"`
 	DS_COR_RACA                   string
 	DS_ESTADO_CIVIL               string
 	DS_NACIONALIDADE              string
@@ -21,6 +21,7 @@ type Politician struct {
 	NM_URNA_CANDIDATO             string
 	NR_TITULO_ELEITORAL_CANDIDATO string
 	SG_UF_NASCIMENTO              string
+	Slug                          string `json:"slug" bson:"slug"`
 	SQ_CANDIDATO                  string
 }
 
@@ -31,3 +32,11 @@ func (c Politician) GetCollectionName() string {
 func (c Politician) Cast() Queryable {
 	return &c
 }
+
+func (c Politician) GetID() string {
+	return c.SQ_CANDIDATO
+}
+
+func (c *Politician) SetSlug(slug string) {
+	c.Slug = slug
+}

politicos/potilical_party.go

@@ -7,6 +7,7 @@ type PoliticalParty struct {
 	FoundedDate string `json:"foundedDate" bson:"foundedDate"`
 	Logo        string `json:"logo" bson:"logo"`
 	Name        string `json:"name" bson:"name"`
+	Slug        string `json:"slug" bson:"slug"`
 	Siglum      string `json:"siglum" bson:"siglum"`
 	TseNumber   string `json:"tseNumber" bson:"tseNumber"`
 	Website     string `json:"website" bson:"website"`
@@ -20,3 +21,11 @@ func (c PoliticalParty) GetCollectionName() string {
 func (c PoliticalParty) Cast() Queryable {
 	return &c
 }
+
+func (c PoliticalParty) GetID() string {
+	return c.Siglum
+}
+
+func (c *PoliticalParty) SetSlug(slug string) {
+	c.Slug = slug
+}

politicos/queryable.go

@@ -6,4 +6,6 @@ package politicos
 type Queryable interface {
 	Cast() Queryable
 	GetCollectionName() string
+	GetID() string
+	SetSlug(slug string)
 }