indexer: extend to improve archive support

- add startTime and endTime to indexer, this way we do not
need the block time estimation.

- add the chainID into the indexer database, so we can identify
the origin chain

- add fromArchive flag to inform the user about the process being
added by the archive

As bonus, clean type package constants.

Signed-off-by: p4u <[email protected]>

api/api_types.go

@@ -34,6 +34,8 @@ type ElectionSummary struct {
 	FinalResults   bool              `json:"finalResults"`
 	Results        [][]*types.BigInt `json:"result,omitempty"`
 	ManuallyEnded  bool              `json:"manuallyEnded"`
+	FromArchive    bool              `json:"fromArchive"`
+	ChainID        string            `json:"chainId"`
 }
 
 // ElectionResults is the struct used to wrap the results of an election

api/helpers.go

@@ -21,15 +21,25 @@ import (
 )
 
 func (a *API) electionSummary(pi *indexertypes.Process) ElectionSummary {
+	startDate := pi.StartDate
+	if startDate.IsZero() {
+		startDate = a.vocinfo.HeightTime(uint64(pi.StartBlock))
+	}
+	endDate := pi.EndDate
+	if endDate.IsZero() {
+		endDate = a.vocinfo.HeightTime(uint64(pi.EndBlock))
+	}
 	return ElectionSummary{
 		ElectionID:     pi.ID,
 		OrganizationID: pi.EntityID,
 		Status:         models.ProcessStatus_name[pi.Status],
-		StartDate:      a.vocinfo.HeightTime(uint64(pi.StartBlock)),
-		EndDate:        a.vocinfo.HeightTime(uint64(pi.EndBlock)),
+		StartDate:      startDate,
+		EndDate:        endDate,
 		FinalResults:   pi.FinalResults,
 		VoteCount:      pi.VoteCount,
 		ManuallyEnded:  pi.EndBlock < pi.StartBlock+pi.BlockCount,
+		ChainID:        pi.ChainID,
+		FromArchive:    pi.FromArchive,
 	}
 }
 
@@ -115,12 +125,8 @@ func convertKeysToCamelInner(val any) any {
 // encodeEVMResultsArgs encodes the arguments for the EVM mimicking the Solidity built-in abi.encode(args...)
 // in this case we encode the organizationId the censusRoot and the results that will be translated in the EVM
 // contract to the corresponding struct{address, bytes32, uint256[][]}
-func encodeEVMResultsArgs(electionId common.Hash,
-	organizationId common.Address,
-	censusRoot common.Hash,
-	sourceContractAddress common.Address,
-	results [][]*types.BigInt,
-) (string, error) {
+func encodeEVMResultsArgs(electionId common.Hash, organizationId common.Address, censusRoot common.Hash,
+	sourceContractAddress common.Address, results [][]*types.BigInt) (string, error) {
 	address, _ := abi.NewType("address", "", nil)
 	bytes32, _ := abi.NewType("bytes32", "", nil)
 	uint256SliceNested, _ := abi.NewType("uint256[][]", "", nil)

service/indexer.go

@@ -24,7 +24,7 @@ func (vs *VocdoniService) VochainIndexer() error {
 
 	if vs.Config.Indexer.ArchiveURL != "" {
 		log.Infow("starting archive retrieval", "path", vs.Config.Indexer.ArchiveURL)
-		go vs.Indexer.StartArchiveRetrival(vs.Storage, vs.Config.Indexer.ArchiveURL)
+		go vs.Indexer.StartArchiveRetrieval(vs.Storage, vs.Config.Indexer.ArchiveURL)
 	}
 
 	return nil

types/consts.go

@@ -1,9 +1,5 @@
 package types
 
-import (
-	"time"
-)
-
 func Bool(b bool) *bool { return &b }
 
 // These exported variables should be treated as constants, to be used in API
@@ -14,152 +10,39 @@ var (
 )
 
 const (
-	// All
-
 	// The mode defines the behaviour of the vocdoninode
 
-	// ModeMiner starts vocdoninode as a miner
+	// ModeMiner starts vocdoninode as a miner.
 	ModeMiner = "miner"
-	// ModeSeed starts vocdoninode as a seed node
+	// ModeSeed starts vocdoninode as a seed node.
 	ModeSeed = "seed"
-	// ModeGateway starts the vocdoninode as a gateway
+	// ModeGateway starts the vocdoninode as a gateway.
 	ModeGateway = "gateway"
-	// ModeCensus starts the vocdoninode as a census only service
+	// ModeCensus starts the vocdoninode as a census only service.
 	ModeCensus = "census"
 
-	// ProcessIDsize is the size of a process id
+	// ProcessIDsize is the size of a process id.
 	ProcessIDsize = 32
-	// EthereumAddressSize is the size of an ethereum address
-	EthereumAddressSize = 20
-
-	// EntityIDsize V2 legacy: in the past we used hash(addr)
-	// this is a temporal work around to support both
-	EntityIDsize = 20
-	// KeyIndexSeparator is the default char used to split keys
-	KeyIndexSeparator = ":"
-	// EthereumConfirmationsThreshold is the minimum amout of blocks
-	// that should pass before considering a tx final
-	EthereumConfirmationsThreshold = 6
-
-	// ENS Domains
-
-	// EntityResolverDomain default entity resolver ENS domain
-	EntityResolverDomain = "entities.voc.eth"
-	// EntityResolverStageDomain is the default entity resolver ENS domain
-	EntityResolverStageDomain = "entities.stg.voc.eth"
-	// EntityResolverDevelopmentDomain is the default entity resolver ENS domain
-	EntityResolverDevelopmentDomain = "entities.dev.voc.eth"
-
-	// ProcessesDomain default process domain
-	ProcessesDomain = "processes.voc.eth"
-	// ProcessesStageDomain stage process domain
-	ProcessesStageDomain = "processes.stg.voc.eth"
-	// ProcessesDevelopmentDomain dev process domain
-	ProcessesDevelopmentDomain = "processes.dev.voc.eth"
-
-	// NamespacesDomain default namespace domain
-	NamespacesDomain = "namespaces.voc.eth"
-	// NamespacesStageDomain stage namespace domain
-	NamespacesStageDomain = "namespaces.stg.voc.eth"
-	// NamespacesDevelopmentDomain dev namespace domain
-	NamespacesDevelopmentDomain = "namespaces.dev.voc.eth"
-
-	// ERC20ProofsDomain default domain for erc20 proofs
-	ERC20ProofsDomain = "erc20.proofs.voc.eth"
-	// ERC20ProofsStageDomain domain for erc20 proofs stage
-	ERC20ProofsStageDomain = "erc20.proofs.stg.voc.eth"
-	// ERC20ProofsDevelopmentDomain domain for erc20 proofs dev
-	ERC20ProofsDevelopmentDomain = "erc20.proofs.dev.voc.eth"
-
-	// GenesisDomain default genesis domain
-	GenesisDomain = "genesis.voc.eth"
-	// GenesisStageDomain stage genesis domain
-	GenesisStageDomain = "genesis.stg.voc.eth"
-	// GenesisDevelopmentDomain dev genesis domain
-	GenesisDevelopmentDomain = "genesis.dev.voc.eth"
-
-	// ResultsDomain default results domain
-	ResultsDomain = "results.voc.eth"
-	// ResultsStageDomain stage results domain
-	ResultsStageDomain = "results.stg.voc.eth"
-	// ResultsDevelopmentDomain dev results domain
-	ResultsDevelopmentDomain = "results.dev.voc.eth"
 
-	// EntityMetaKey is the key of an ENS text record for the entity metadata
-	EntityMetaKey = "vnd.vocdoni.meta"
-
-	// EthereumReadTimeout is the max amount of time for reading anything on
-	// the Ethereum network to wait until canceling it's context
-	EthereumReadTimeout = 1 * time.Minute
-	// EthereumWriteTimeout is the max amount of time for writing anything on
-	// the Ethereum network to wait until canceling it's context
-	EthereumWriteTimeout = 1 * time.Minute
-	// EthereumDialMaxRetry is the max number of attempts an ethereum client will
-	// make in order to dial to an endpoint before considering the endpoint unreachable
-	EthereumDialMaxRetry = 10
-
-	// Indexer
+	// EthereumAddressSize is the size of an ethereum address.
+	EthereumAddressSize = 20
 
-	// IndexerLiveProcessPrefix is used for sotring temporary results on live
-	IndexerLiveProcessPrefix = byte(0x21)
-	// IndexerEntityPrefix is the prefix for the storage entity keys
-	IndexerEntityPrefix = byte(0x22)
-	// IndexerResultsPrefix is the prefix of the storage results summary keys
-	IndexerResultsPrefix = byte(0x24)
-	// IndexerProcessEndingPrefix is the prefix for keep track of the processes ending
-	// on a specific block
-	IndexerProcessEndingPrefix = byte(0x25)
+	// EntityIDsize is the size of an entity id (ethereum address).
+	EntityIDsize = EthereumAddressSize
 
-	// ArchiveURL is the default URL where the archive is retrieved from
+	// ArchiveURL is the default URL where the archive is retrieved from.
 	ArchiveURL = "/ipns/k2k4r8mdn544n7f8nprwqeo27jr1v1unsu74th57s1j8mumjck7y7cbz"
 
-	// Vochain
+	// DefaultBlockTimeSeconds is the default block time in seconds.
+	DefaultBlockTimeSeconds = 12
 
-	// PetitionSign contains the string that needs to match with the received vote type
-	// for petition-sign
-	PetitionSign = "petition-sign"
-	// PollVote contains the string that needs to match with the received vote type for poll-vote
-	PollVote = "poll-vote"
-	// EncryptedPoll contains the string that needs to match with the received vote type
-	// for encrypted-poll
-	EncryptedPoll = "encrypted-poll"
-	// SnarkVote contains the string that needs to match with the received vote type for snark-vote
-	SnarkVote = "snark-vote"
-
-	// KeyKeeper
-
-	// KeyKeeperMaxKeyIndex is the maxim number of allowed encryption keys
+	// KeyKeeperMaxKeyIndex is the maxim number of allowed encryption keys.
 	KeyKeeperMaxKeyIndex = 16
 
-	// List of transition names
-
-	TxVote              = "vote"
-	TxNewProcess        = "newProcess"
-	TxCancelProcess     = "cancelProcess" // legacy
-	TxSetProcess        = "setProcess"
-	TxAddValidator      = "addValidator"
-	TxRemoveValidator   = "removeValidator"
-	TxAddProcessKeys    = "addProcessKeys"
-	TxRevealProcessKeys = "revealProcessKeys"
-
-	// ProcessesContractMaxProcessMode represents the max value that a uint8 can have
-	// with the current smart contract bitmask describing the supported process mode
-	ProcessesContractMaxProcessMode = 31
 	// ProcessesContractMaxEnvelopeType represents the max value that a uint8 can have
-	// with the current smart contract bitmask describing the supported envelope types
+	// with the current smart contract bitmask describing the supported envelope types.
 	ProcessesContractMaxEnvelopeType = 31
 
-	// TODO: @jordipainan this values are tricky
-
-	// ProcessesContractMinBlockCount represents the minimum number of vochain blocks
-	// that a process should last
-	ProcessesContractMinBlockCount = 2
-
-	// ProcessesParamsSignatureSize represents the size of a signature on ethereum
-	ProcessesParamsSignatureSize = 32
-
-	VochainWsReadLimit = 20 << 20 // tendermint requires 20 MiB minimum
-	Web3WsReadLimit    = 5 << 20  // go-ethereum accepts maximum 5 MiB
-
+	// MaxURLLength is the maximum length of a URL string used in the protocol.
 	MaxURLLength = 2083
 )

vochain/indexer/archive.go

@@ -40,7 +40,6 @@ func (idx *Indexer) ImportArchive(archive []*ArchiveProcess) ([]*ArchiveProcess,
 		return nil, err
 	}
 	defer tx.Rollback()
-	height := idx.App.State.CurrentHeight()
 	queries := indexerdb.New(tx)
 	added := []*ArchiveProcess{}
 	for _, p := range archive {
@@ -61,17 +60,33 @@ func (idx *Indexer) ImportArchive(archive []*ArchiveProcess) ([]*ArchiveProcess,
 		} else {
 			continue
 		}
-		creationTime := time.Now()
-		if p.StartDate != nil {
-			creationTime = *p.StartDate
+
+		// For backward compatibility, we try to fetch the start/end date from multiple sources.
+		// If not found, we calculate them from the block count and the default block time.
+		startDate := p.ProcessInfo.StartDate
+		if startDate.IsZero() {
+			if p.StartDate != nil {
+				startDate = *p.StartDate
+			} else {
+				// Calculate startDate equal to time.Now() minus defaultBlockTime*p.ProcessInfo.BlockCount
+				startDate = time.Now().Add(-types.DefaultBlockTimeSeconds * time.Duration(p.ProcessInfo.BlockCount))
+			}
+		}
+		endDate := p.ProcessInfo.EndDate
+		if endDate.IsZero() {
+			// Calculate endDate equal to startDate plus defaultBlockTime*p.ProcessInfo.BlockCount
+			endDate = startDate.Add(types.DefaultBlockTimeSeconds * time.Duration(p.ProcessInfo.BlockCount))
 		}
+
 		// Create and store process in the indexer database
 		procParams := indexerdb.CreateProcessParams{
 			ID:                nonNullBytes(p.ProcessInfo.ID),
 			EntityID:          nonNullBytes(p.ProcessInfo.EntityID),
-			StartBlock:        int64(height),
-			EndBlock:          int64(height + 1),
-			BlockCount:        int64(1),
+			StartBlock:        int64(p.ProcessInfo.StartBlock),
+			StartDate:         startDate,
+			EndBlock:          int64(p.ProcessInfo.EndBlock),
+			EndDate:           endDate,
+			BlockCount:        int64(p.ProcessInfo.BlockCount),
 			HaveResults:       p.ProcessInfo.HaveResults,
 			FinalResults:      p.ProcessInfo.FinalResults,
 			CensusRoot:        nonNullBytes(p.ProcessInfo.CensusRoot),
@@ -85,13 +100,16 @@ func (idx *Indexer) ImportArchive(archive []*ArchiveProcess) ([]*ArchiveProcess,
 			VoteOpts:          indexertypes.EncodeProtoJSON(p.ProcessInfo.VoteOpts),
 			PrivateKeys:       indexertypes.EncodeJSON(p.ProcessInfo.PrivateKeys),
 			PublicKeys:        indexertypes.EncodeJSON(p.ProcessInfo.PublicKeys),
-			CreationTime:      creationTime,
+			CreationTime:      time.Now(),
 			SourceBlockHeight: int64(p.ProcessInfo.SourceBlockHeight),
 			SourceNetworkID:   int64(models.SourceNetworkId_value[p.ProcessInfo.SourceNetworkId]),
 			Metadata:          p.ProcessInfo.Metadata,
 			ResultsVotes:      indexertypes.EncodeJSON(p.Results.Votes),
 			VoteCount:         int64(p.ProcessInfo.VoteCount),
+			ChainID:           p.ChainID,
+			FromArchive:       true,
 		}
+
 		if _, err := queries.CreateProcess(context.TODO(), procParams); err != nil {
 			return nil, fmt.Errorf("create archive process: %w", err)
 		}
@@ -100,9 +118,9 @@ func (idx *Indexer) ImportArchive(archive []*ArchiveProcess) ([]*ArchiveProcess,
 	return added, tx.Commit()
 }
 
-// StartArchiveRetrival starts the archive retrieval process. It is a blocking function that runs continuously.
+// StartArchiveRetrieval starts the archive retrieval process. It is a blocking function that runs continuously.
 // Retrieves the archive directory from the storage and imports the processes into the indexer database.
-func (idx *Indexer) StartArchiveRetrival(storage data.Storage, archiveURL string) {
+func (idx *Indexer) StartArchiveRetrieval(storage data.Storage, archiveURL string) {
 	for {
 		ctx, cancel := context.WithTimeout(context.Background(), timeoutArchiveRetrieval)
 		dirMap, err := storage.RetrieveDir(ctx, archiveURL, marxArchiveFileSize)

vochain/indexer/archive_test.go

@@ -27,9 +27,13 @@ func TestImportArchive(t *testing.T) {
 	qt.Assert(t, process1.Results().Votes[0][0].MathBigInt().Int64(), qt.Equals, int64(342))
 	qt.Assert(t, process1.Results().Votes[0][1].MathBigInt().Int64(), qt.Equals, int64(365))
 	qt.Assert(t, process1.Results().Votes[0][2].MathBigInt().Int64(), qt.Equals, int64(21))
-	// TODO: qt.Assert(t, process1.Results().Weight.MathBigInt().Int64(), qt.Equals, int64(342+365+21))
+	qt.Assert(t, process1.StartDate, qt.DeepEquals, *archiveProcess1.StartDate)
+	// check that endDate is set after startDate
+	qt.Assert(t, process1.EndDate.After(process1.StartDate), qt.Equals, true)
 }
 
+// This is an old archive process format, we check backwards compatibility.
+// At some point we should update this format to the new one.
 var testArchiveProcess1 = `
 {
   "chainId": "vocdoni-stage-8",