ingest: service created #14
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,2 +1,3 @@ | ||
| # Visual Studio Code | ||
| .vscode | ||
| captive-core*/ | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,101 @@ | ||
| package cmd | ||
|
|
||
| import ( | ||
| "go/types" | ||
|
|
||
| _ "github.com/lib/pq" | ||
| "github.com/spf13/cobra" | ||
| "github.com/stellar/go/network" | ||
| "github.com/stellar/go/support/config" | ||
| "github.com/stellar/go/support/log" | ||
| "github.com/stellar/wallet-backend/internal/ingest" | ||
| ) | ||
|
|
||
| type ingestCmd struct{} | ||
|
|
||
| func (c *ingestCmd) Command() *cobra.Command { | ||
| cfg := ingest.Configs{} | ||
| cfgOpts := config.ConfigOptions{ | ||
| { | ||
| Name: "database-url", | ||
| Usage: "Database connection URL.", | ||
| OptType: types.String, | ||
| ConfigKey: &cfg.DatabaseURL, | ||
| FlagDefault: "postgres://postgres@localhost:5432/wallet-backend?sslmode=disable", | ||
| Required: true, | ||
| }, | ||
| { | ||
| Name: "network-passphrase", | ||
| Usage: "Stellar Network Passphrase to connect.", | ||
| OptType: types.String, | ||
| ConfigKey: &cfg.NetworkPassphrase, | ||
| FlagDefault: network.TestNetworkPassphrase, | ||
| Required: true, | ||
| }, | ||
| { | ||
| Name: "captive-core-bin-path", | ||
| Usage: "Path to Captive Core's binary file.", | ||
| OptType: types.String, | ||
| ConfigKey: &cfg.CaptiveCoreBinPath, | ||
|
There was a problem hiding this comment. I was thinking about adding a There was a problem hiding this comment. Good one! c81a985 |
||
| FlagDefault: "/usr/local/bin/stellar-core", | ||
|
There was a problem hiding this comment. I wonder if this one couldn't also point to a local directory like the There was a problem hiding this comment. Whenever you install captive core locally using these instructions, |
||
| Required: true, | ||
| }, | ||
| { | ||
| Name: "captive-core-config-dir", | ||
| Usage: "Path to Captive Core's configuration files directory.", | ||
| OptType: types.String, | ||
| ConfigKey: &cfg.CaptiveCoreConfigDir, | ||
|
There was a problem hiding this comment. Same here (custom set value for folders value). There was a problem hiding this comment. Good one! c81a985 |
||
| FlagDefault: "./internal/ingest/config", | ||
| Required: true, | ||
| }, | ||
| { | ||
| Name: "ledger-cursor-name", | ||
| Usage: "Name of last synced ledger cursor. Attention: there should never be more than one container running with a same cursor name.", | ||
|
There was a problem hiding this comment. Could you elaborate on it? Is this value an enum or should it be a unique value? We should make it more clear and, also remove the There was a problem hiding this comment. It's not an enum, it's just an unique string value used as the key in the There was a problem hiding this comment. I've made the description more clear, made it required and removed the default value. Good call, this will make people more aware that this has to be carefully configured when running parallel ingestion. |
||
| OptType: types.String, | ||
| ConfigKey: &cfg.LedgerCursorName, | ||
| FlagDefault: "last_synced_ledger", | ||
| Required: false, | ||
| }, | ||
| { | ||
| Name: "start", | ||
| Usage: "Ledger number from which ingestion should start. When not present, ingestion will resume from last synced ledger.", | ||
| OptType: types.Int, | ||
| ConfigKey: &cfg.StartLedger, | ||
| FlagDefault: 0, | ||
| Required: false, | ||
| }, | ||
| { | ||
| Name: "end", | ||
| Usage: "Ledger number up to which ingestion should run. When not present, ingestion run indefinitely (live ingestion requires it to be empty).", | ||
| OptType: types.Int, | ||
| ConfigKey: &cfg.EndLedger, | ||
| FlagDefault: 0, | ||
| Required: false, | ||
| }, | ||
| } | ||
|
|
||
| cmd := &cobra.Command{ | ||
| Use: "ingest", | ||
| Short: "Run Ingestion service", | ||
| PersistentPreRun: func(_ *cobra.Command, _ []string) { | ||
| cfgOpts.Require() | ||
| if err := cfgOpts.SetValues(); err != nil { | ||
| log.Fatalf("Error setting values of config options: %s", err.Error()) | ||
| } | ||
| }, | ||
| Run: func(_ *cobra.Command, _ []string) { | ||
| c.Run(cfg) | ||
| }, | ||
| } | ||
| if err := cfgOpts.Init(cmd); err != nil { | ||
| log.Fatalf("Error initializing a config option: %s", err.Error()) | ||
| } | ||
| return cmd | ||
| } | ||
|
|
||
| func (c *ingestCmd) Run(cfg ingest.Configs) { | ||
| err := ingest.Ingest(cfg) | ||
| if err != nil { | ||
| log.Fatalf("Error running Ingest: %s", err.Error()) | ||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Is this name correct?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yes, whenever you run captive core it creates a folder named like
captive-core-9f8g7a9fin the application's root folder to store the ledger files it downloads. The "*" is to capture the random name given to the folder.