From 008df20a57b0ba72f282933e42bd5638f1a836b5 Mon Sep 17 00:00:00 2001 From: dermanyang Date: Wed, 15 Jan 2025 12:13:35 -0600 Subject: [PATCH 1/5] add content --- apps/nextra/next.config.mjs | 6 ++ .../indexer-sdk/documentation/_meta.tsx | 3 + .../documentation/version-tracking.mdx | 67 +++++++++++++++++++ 3 files changed, 76 insertions(+) create mode 100644 apps/nextra/pages/en/build/indexer/indexer-sdk/documentation/version-tracking.mdx diff --git a/apps/nextra/next.config.mjs b/apps/nextra/next.config.mjs index 6b65ad29e..d7d7891db 100644 --- a/apps/nextra/next.config.mjs +++ b/apps/nextra/next.config.mjs @@ -429,6 +429,12 @@ export default withBundleAnalyzer( "/en/build/indexer/indexer-sdk/documentation/connect-steps", permanent: true, }, + { + source: "/indexer/indexer-sdk/documentation/version-tracking", + destination: + "/en/build/indexer/indexer-sdk/documentation/version-tracking", + permanent: true, + }, { source: "/indexer/txn-stream/labs-hosted", destination: "/en/build/indexer/api/labs-hosted", diff --git a/apps/nextra/pages/en/build/indexer/indexer-sdk/documentation/_meta.tsx b/apps/nextra/pages/en/build/indexer/indexer-sdk/documentation/_meta.tsx index 2ea78a2d2..b23423f60 100644 --- a/apps/nextra/pages/en/build/indexer/indexer-sdk/documentation/_meta.tsx +++ b/apps/nextra/pages/en/build/indexer/indexer-sdk/documentation/_meta.tsx @@ -12,4 +12,7 @@ export default { "connect-steps": { title: "Connecting Steps", }, + "version-tracking": { + title: "Version Tracking, Restart Behavior, and Backfilling", + }, }; diff --git a/apps/nextra/pages/en/build/indexer/indexer-sdk/documentation/version-tracking.mdx b/apps/nextra/pages/en/build/indexer/indexer-sdk/documentation/version-tracking.mdx new file mode 100644 index 000000000..ada23cf32 --- /dev/null +++ b/apps/nextra/pages/en/build/indexer/indexer-sdk/documentation/version-tracking.mdx @@ -0,0 +1,67 @@ +--- +title: "Version Tracking, Restart Behavior, and Backfilling" +--- + +# Version Tracking, Restart Behavior, and Backfilling + +## Version Tracking +The `VersionTrackerStep` is a [common step in the SDK](https://github.com/aptos-labs/aptos-indexer-processor-sdk/blob/main/aptos-indexer-processors-sdk/sdk/src/common_steps/version_tracker_step.rs#L29C12-L29C30) that checkpoints indexer progress. The struct +requires implementation of the [`ProcessorStatusSaver` trait](https://github.com/aptos-labs/aptos-indexer-processor-sdk/blob/main/aptos-indexer-processors-sdk/sdk/src/common_steps/version_tracker_step.rs#L16) and can be used with the same [`.connect_to()`](https://github.com/aptos-labs/aptos-indexer-processors/blob/main/rust/sdk-processor/src/processors/events_processor.rs#L117) method as other steps. Upon a successfully processed batch, +the `VersionTrackerStep` will [call](https://github.com/aptos-labs/aptos-indexer-processor-sdk/blob/main/aptos-indexer-processors-sdk/sdk/src/common_steps/version_tracker_step.rs#L57) the trait implementation of `save_processor_status()`. + +### `ProcessorStatusSaver` +The `ProcessorStatusSaver` trait requires the implementation of the method `save_processor_status` with the following signature: + +```rust +async fn save_processor_status( + &self, + last_success_batch: &TransactionContext<()>, + ) -> Result<(), ProcessorError>; +``` +This method is where checkpointing should be written. Examples can be found [here](https://github.com/aptos-labs/aptos-indexer-processors/blob/main/rust/sdk-processor/src/steps/common/processor_status_saver.rs#L23). +Note that it is possible to checkpoint progress in different ways by using enums. [This example](https://github.com/aptos-labs/aptos-indexer-processors/blob/main/rust/sdk-processor/src/steps/common/processor_status_saver.rs#L131) inserts into Postgres using a simple [processor_status model](https://github.com/aptos-labs/aptos-indexer-processors/blob/main/rust/processor/src/db/postgres/models/processor_status.rs). + +## Restart Behavior +Now that the processor successfully writes to the chosen store for checkpointing, upon restarting it needs to retrieve the latest successful version from that store. Here is an example of a `get_starting_version()` method that returns the latest processed version saved. This `starting_version: u64` can then be used as below. If there is no checkpoint, the processor will start from the beginning of the chain. + +```rust + let transaction_stream = TransactionStreamStep::new(TransactionStreamConfig { + starting_version: Some(starting_version), + ..self.config.transaction_stream_config.clone() + }) + .await?; +``` + +## Backfilling +The `IndexerProcessorConfig`, `ProcessorStatusSaver` and `get_starting_version()` can all be modified to allow for convenient backfills. Without these changes, it is difficult to run a live processor at the latest transaction version as well as a backfill processor. + +### Updates to Config +[Add an additional field](https://github.com/aptos-labs/aptos-indexer-processors/blob/main/rust/sdk-processor/src/config/indexer_processor_config.rs#L44) on your `IndexerProcessorConfig` for a `BackfillConfig` which contains a single field, `backfill_alias` . The introduction of this field will be used to determine whether the processor is a backfill processor or regular one. + +```rust +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct IndexerProcessorConfig { + pub processor_config: ProcessorConfig, + pub transaction_stream_config: TransactionStreamConfig, + pub db_config: DbConfig, + pub backfill_config: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BackfillConfig { + pub backfill_alias: String, +} +``` + +### Backfill Processor Status Table +To avoid write conflicts with the regular `processor_status` table, introduce a `backfill_processor_status_table` that will use `backfill_alias` as the primary key instead of `processor_name`. +Expand on this [implementation](https://github.com/aptos-labs/aptos-indexer-processors/blob/main/rust/sdk-processor/src/db/common/models/backfill_processor_status.rs). This model introduces a new state, `BackfillStatus`, which is either `InProgress` or `Complete` which will determine the backfilling restart behavior as expected. + + +### Updates to ProcessorStatusSaver +Expand your `ProcessorStatusSaverEnum` to include a `Backfill` variant that extracts the `backfill_alias` from the `BackfillConfig`, and the `backfill_start_version` `backfill_end_version` from `IndexerProcessorConfig.transaction_stream_config` [like this](https://github.com/aptos-labs/aptos-indexer-processors/blob/main/rust/sdk-processor/src/steps/common/processor_status_saver.rs#L27). Update the corresponding write query to write to the new `backfill_processor_status` table. + +### Updates to `get_starting_version()` +Add a [statement](https://github.com/aptos-labs/aptos-indexer-processors/blob/main/rust/sdk-processor/src/utils/starting_version.rs#L143) in your `get_starting_version` method to query the `backfill_processor_status_table` when the `BackfillConfig` field is present in `IndexerProcessorConfig` . + From dcc2f6d24e6e765ba1a5cd97a02c6e41f49580ae Mon Sep 17 00:00:00 2001 From: dermanyang Date: Wed, 15 Jan 2025 12:20:28 -0600 Subject: [PATCH 2/5] formatting --- .../src/utils/mdast/mdast.test.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/aptos-nextra-components/src/utils/mdast/mdast.test.ts b/packages/aptos-nextra-components/src/utils/mdast/mdast.test.ts index 167c1aa9e..858dc87e2 100644 --- a/packages/aptos-nextra-components/src/utils/mdast/mdast.test.ts +++ b/packages/aptos-nextra-components/src/utils/mdast/mdast.test.ts @@ -7,7 +7,9 @@ test("keyRotation.md parse and fix AST", () => { markdownToMdx(tree); const markdown = astToMarkdown(tree); expect(markdown).toEqual( - astToMarkdown(readFileAsTree("./examples/keyRotation.expect.md")).toString() + astToMarkdown( + readFileAsTree("./examples/keyRotation.expect.md"), + ).toString(), ); }); @@ -18,8 +20,8 @@ test("account_snippet.md parse and fix AST", () => { const markdown = astToMarkdown(tree); expect(markdown).toEqual( astToMarkdown( - readFileAsTree("./examples/account_snippet.expect.md") - ).toString() + readFileAsTree("./examples/account_snippet.expect.md"), + ).toString(), ); }); @@ -29,8 +31,8 @@ test("vector_snippet.md parse and fix AST", () => { const markdown = astToMarkdown(tree); expect(markdown).toEqual( astToMarkdown( - readFileAsTree("./examples/vector_snippet.expect.md") - ).toString() + readFileAsTree("./examples/vector_snippet.expect.md"), + ).toString(), ); }); From b62fb588a66ed7250d0ab577de8a93759188358e Mon Sep 17 00:00:00 2001 From: dermanyang Date: Wed, 15 Jan 2025 12:43:05 -0600 Subject: [PATCH 3/5] Change some headers and remove new lines --- .../indexer-sdk/documentation/_meta.tsx | 2 +- .../documentation/version-tracking.mdx | 21 ++++++++----------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/apps/nextra/pages/en/build/indexer/indexer-sdk/documentation/_meta.tsx b/apps/nextra/pages/en/build/indexer/indexer-sdk/documentation/_meta.tsx index b23423f60..ae4a78d5c 100644 --- a/apps/nextra/pages/en/build/indexer/indexer-sdk/documentation/_meta.tsx +++ b/apps/nextra/pages/en/build/indexer/indexer-sdk/documentation/_meta.tsx @@ -13,6 +13,6 @@ export default { title: "Connecting Steps", }, "version-tracking": { - title: "Version Tracking, Restart Behavior, and Backfilling", + title: "Version Tracking", }, }; diff --git a/apps/nextra/pages/en/build/indexer/indexer-sdk/documentation/version-tracking.mdx b/apps/nextra/pages/en/build/indexer/indexer-sdk/documentation/version-tracking.mdx index ada23cf32..78ce08dcd 100644 --- a/apps/nextra/pages/en/build/indexer/indexer-sdk/documentation/version-tracking.mdx +++ b/apps/nextra/pages/en/build/indexer/indexer-sdk/documentation/version-tracking.mdx @@ -1,15 +1,13 @@ --- -title: "Version Tracking, Restart Behavior, and Backfilling" +title: "Version Tracking" --- # Version Tracking, Restart Behavior, and Backfilling ## Version Tracking -The `VersionTrackerStep` is a [common step in the SDK](https://github.com/aptos-labs/aptos-indexer-processor-sdk/blob/main/aptos-indexer-processors-sdk/sdk/src/common_steps/version_tracker_step.rs#L29C12-L29C30) that checkpoints indexer progress. The struct -requires implementation of the [`ProcessorStatusSaver` trait](https://github.com/aptos-labs/aptos-indexer-processor-sdk/blob/main/aptos-indexer-processors-sdk/sdk/src/common_steps/version_tracker_step.rs#L16) and can be used with the same [`.connect_to()`](https://github.com/aptos-labs/aptos-indexer-processors/blob/main/rust/sdk-processor/src/processors/events_processor.rs#L117) method as other steps. Upon a successfully processed batch, -the `VersionTrackerStep` will [call](https://github.com/aptos-labs/aptos-indexer-processor-sdk/blob/main/aptos-indexer-processors-sdk/sdk/src/common_steps/version_tracker_step.rs#L57) the trait implementation of `save_processor_status()`. +The `VersionTrackerStep` is a [common step in the SDK](https://github.com/aptos-labs/aptos-indexer-processor-sdk/blob/main/aptos-indexer-processors-sdk/sdk/src/common_steps/version_tracker_step.rs#L29C12-L29C30) that checkpoints indexer progress. The struct requires implementation of the [`ProcessorStatusSaver` trait](https://github.com/aptos-labs/aptos-indexer-processor-sdk/blob/main/aptos-indexer-processors-sdk/sdk/src/common_steps/version_tracker_step.rs#L16) and can be used with the same [`.connect_to()`](https://github.com/aptos-labs/aptos-indexer-processors/blob/main/rust/sdk-processor/src/processors/events_processor.rs#L117) method as other steps. Upon a successfully processed batch, the `VersionTrackerStep` will [call](https://github.com/aptos-labs/aptos-indexer-processor-sdk/blob/main/aptos-indexer-processors-sdk/sdk/src/common_steps/version_tracker_step.rs#L57) the trait implementation of `save_processor_status()`. -### `ProcessorStatusSaver` +### ProcessorStatusSaver The `ProcessorStatusSaver` trait requires the implementation of the method `save_processor_status` with the following signature: ```rust @@ -18,8 +16,8 @@ async fn save_processor_status( last_success_batch: &TransactionContext<()>, ) -> Result<(), ProcessorError>; ``` -This method is where checkpointing should be written. Examples can be found [here](https://github.com/aptos-labs/aptos-indexer-processors/blob/main/rust/sdk-processor/src/steps/common/processor_status_saver.rs#L23). -Note that it is possible to checkpoint progress in different ways by using enums. [This example](https://github.com/aptos-labs/aptos-indexer-processors/blob/main/rust/sdk-processor/src/steps/common/processor_status_saver.rs#L131) inserts into Postgres using a simple [processor_status model](https://github.com/aptos-labs/aptos-indexer-processors/blob/main/rust/processor/src/db/postgres/models/processor_status.rs). +This method is where checkpointing should be written. Examples can be found [here](https://github.com/aptos-labs/aptos-indexer-processors/blob/main/rust/sdk-processor/src/steps/common/processor_status_saver.rs#L23). Note that it is possible to checkpoint progress in different ways by using enums. [This example](https://github.com/aptos-labs/aptos-indexer-processors/blob/main/rust/sdk-processor/src/steps/common/processor_status_saver.rs#L131) inserts into Postgres using a simple [processor_status model](https://github.com/aptos-labs/aptos-indexer-processors/blob/main/rust/processor/src/db/postgres/models/processor_status.rs). + ## Restart Behavior Now that the processor successfully writes to the chosen store for checkpointing, upon restarting it needs to retrieve the latest successful version from that store. Here is an example of a `get_starting_version()` method that returns the latest processed version saved. This `starting_version: u64` can then be used as below. If there is no checkpoint, the processor will start from the beginning of the chain. @@ -32,11 +30,12 @@ Now that the processor successfully writes to the chosen store for checkpointing .await?; ``` + ## Backfilling The `IndexerProcessorConfig`, `ProcessorStatusSaver` and `get_starting_version()` can all be modified to allow for convenient backfills. Without these changes, it is difficult to run a live processor at the latest transaction version as well as a backfill processor. ### Updates to Config -[Add an additional field](https://github.com/aptos-labs/aptos-indexer-processors/blob/main/rust/sdk-processor/src/config/indexer_processor_config.rs#L44) on your `IndexerProcessorConfig` for a `BackfillConfig` which contains a single field, `backfill_alias` . The introduction of this field will be used to determine whether the processor is a backfill processor or regular one. +[Add an additional field](https://github.com/aptos-labs/aptos-indexer-processors/blob/main/rust/sdk-processor/src/config/indexer_processor_config.rs#L44) on your `IndexerProcessorConfig` for a `BackfillConfig` which contains a single field, `backfill_alias`. The introduction of this field will be used to determine whether the processor is a backfill processor or regular one. ```rust #[derive(Clone, Debug, Deserialize, Serialize)] @@ -55,13 +54,11 @@ pub struct BackfillConfig { ``` ### Backfill Processor Status Table -To avoid write conflicts with the regular `processor_status` table, introduce a `backfill_processor_status_table` that will use `backfill_alias` as the primary key instead of `processor_name`. -Expand on this [implementation](https://github.com/aptos-labs/aptos-indexer-processors/blob/main/rust/sdk-processor/src/db/common/models/backfill_processor_status.rs). This model introduces a new state, `BackfillStatus`, which is either `InProgress` or `Complete` which will determine the backfilling restart behavior as expected. - +To avoid write conflicts with the regular `processor_status` table, introduce a `backfill_processor_status_table` that will use `backfill_alias` as the primary key instead of `processor_name`. Expand on this [implementation](https://github.com/aptos-labs/aptos-indexer-processors/blob/main/rust/sdk-processor/src/db/common/models/backfill_processor_status.rs). This model introduces a new state, `BackfillStatus`, which is either `InProgress` or `Complete` which will determine the backfilling restart behavior as expected. ### Updates to ProcessorStatusSaver Expand your `ProcessorStatusSaverEnum` to include a `Backfill` variant that extracts the `backfill_alias` from the `BackfillConfig`, and the `backfill_start_version` `backfill_end_version` from `IndexerProcessorConfig.transaction_stream_config` [like this](https://github.com/aptos-labs/aptos-indexer-processors/blob/main/rust/sdk-processor/src/steps/common/processor_status_saver.rs#L27). Update the corresponding write query to write to the new `backfill_processor_status` table. -### Updates to `get_starting_version()` +### Updates to get_starting_version Add a [statement](https://github.com/aptos-labs/aptos-indexer-processors/blob/main/rust/sdk-processor/src/utils/starting_version.rs#L143) in your `get_starting_version` method to query the `backfill_processor_status_table` when the `BackfillConfig` field is present in `IndexerProcessorConfig` . From 1c8891f90fc892f1ca881c5cc59d1eeeb8e22277 Mon Sep 17 00:00:00 2001 From: dermanyang Date: Wed, 15 Jan 2025 12:55:21 -0600 Subject: [PATCH 4/5] Change wording --- .../indexer-sdk/documentation/version-tracking.mdx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/nextra/pages/en/build/indexer/indexer-sdk/documentation/version-tracking.mdx b/apps/nextra/pages/en/build/indexer/indexer-sdk/documentation/version-tracking.mdx index 78ce08dcd..f8327f0d7 100644 --- a/apps/nextra/pages/en/build/indexer/indexer-sdk/documentation/version-tracking.mdx +++ b/apps/nextra/pages/en/build/indexer/indexer-sdk/documentation/version-tracking.mdx @@ -16,11 +16,11 @@ async fn save_processor_status( last_success_batch: &TransactionContext<()>, ) -> Result<(), ProcessorError>; ``` -This method is where checkpointing should be written. Examples can be found [here](https://github.com/aptos-labs/aptos-indexer-processors/blob/main/rust/sdk-processor/src/steps/common/processor_status_saver.rs#L23). Note that it is possible to checkpoint progress in different ways by using enums. [This example](https://github.com/aptos-labs/aptos-indexer-processors/blob/main/rust/sdk-processor/src/steps/common/processor_status_saver.rs#L131) inserts into Postgres using a simple [processor_status model](https://github.com/aptos-labs/aptos-indexer-processors/blob/main/rust/processor/src/db/postgres/models/processor_status.rs). +This method is where checkpointing should be written. Examples can be found [here](https://github.com/aptos-labs/aptos-indexer-processors/blob/main/rust/sdk-processor/src/steps/common/processor_status_saver.rs#L23). It is possible to checkpoint progress in different ways by using enums. [This example](https://github.com/aptos-labs/aptos-indexer-processors/blob/main/rust/sdk-processor/src/steps/common/processor_status_saver.rs#L131) inserts into Postgres using a simple [`processor_status` model](https://github.com/aptos-labs/aptos-indexer-processors/blob/main/rust/processor/src/db/postgres/models/processor_status.rs). ## Restart Behavior -Now that the processor successfully writes to the chosen store for checkpointing, upon restarting it needs to retrieve the latest successful version from that store. Here is an example of a `get_starting_version()` method that returns the latest processed version saved. This `starting_version: u64` can then be used as below. If there is no checkpoint, the processor will start from the beginning of the chain. +Now that the processor successfully writes to the chosen store for version tracking, upon restarting it needs to retrieve the latest successful version from that store. [Here is an example](https://github.com/aptos-labs/aptos-indexer-processors/blob/29f4e9c2266db11cbec2177c17fc210d4c687d32/rust/sdk-processor/src/utils/starting_version.rs#L26) of a `get_starting_version()` method that returns the latest processed version saved. This `starting_version: u64` can then be used as below. If there is no checkpoint, the processor will start from the beginning of the chain. ```rust let transaction_stream = TransactionStreamStep::new(TransactionStreamConfig { @@ -32,7 +32,7 @@ Now that the processor successfully writes to the chosen store for checkpointing ## Backfilling -The `IndexerProcessorConfig`, `ProcessorStatusSaver` and `get_starting_version()` can all be modified to allow for convenient backfills. Without these changes, it is difficult to run a live processor at the latest transaction version as well as a backfill processor. +To enable backfilling, `IndexerProcessorConfig`, `ProcessorStatusSaver` and `get_starting_version()` need some updates. Without these changes, it is difficult to run a live processor at the latest transaction version as well as a backfill processor. ### Updates to Config [Add an additional field](https://github.com/aptos-labs/aptos-indexer-processors/blob/main/rust/sdk-processor/src/config/indexer_processor_config.rs#L44) on your `IndexerProcessorConfig` for a `BackfillConfig` which contains a single field, `backfill_alias`. The introduction of this field will be used to determine whether the processor is a backfill processor or regular one. @@ -54,7 +54,7 @@ pub struct BackfillConfig { ``` ### Backfill Processor Status Table -To avoid write conflicts with the regular `processor_status` table, introduce a `backfill_processor_status_table` that will use `backfill_alias` as the primary key instead of `processor_name`. Expand on this [implementation](https://github.com/aptos-labs/aptos-indexer-processors/blob/main/rust/sdk-processor/src/db/common/models/backfill_processor_status.rs). This model introduces a new state, `BackfillStatus`, which is either `InProgress` or `Complete` which will determine the backfilling restart behavior as expected. +To avoid write conflicts with the regular `processor_status` table, introduce a `backfill_processor_status_table` that will use `backfill_alias` as the primary key instead of `processor_name`. Expand on this [implementation](https://github.com/aptos-labs/aptos-indexer-processors/blob/main/rust/sdk-processor/src/db/common/models/backfill_processor_status.rs). This model introduces a new state, `BackfillStatus`, which is either `InProgress` or `Complete` which will determine the backfilling restart behavior. ### Updates to ProcessorStatusSaver Expand your `ProcessorStatusSaverEnum` to include a `Backfill` variant that extracts the `backfill_alias` from the `BackfillConfig`, and the `backfill_start_version` `backfill_end_version` from `IndexerProcessorConfig.transaction_stream_config` [like this](https://github.com/aptos-labs/aptos-indexer-processors/blob/main/rust/sdk-processor/src/steps/common/processor_status_saver.rs#L27). Update the corresponding write query to write to the new `backfill_processor_status` table. From 04358b09b81977fb9fdc1d95f71a6b40abbd51e4 Mon Sep 17 00:00:00 2001 From: dermanyang Date: Wed, 22 Jan 2025 11:26:19 -0800 Subject: [PATCH 5/5] Update reference to example repo and add sections on write conflicts + config.yaml --- .../documentation/version-tracking.mdx | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/apps/nextra/pages/en/build/indexer/indexer-sdk/documentation/version-tracking.mdx b/apps/nextra/pages/en/build/indexer/indexer-sdk/documentation/version-tracking.mdx index f8327f0d7..d8d100a1d 100644 --- a/apps/nextra/pages/en/build/indexer/indexer-sdk/documentation/version-tracking.mdx +++ b/apps/nextra/pages/en/build/indexer/indexer-sdk/documentation/version-tracking.mdx @@ -35,7 +35,7 @@ Now that the processor successfully writes to the chosen store for version track To enable backfilling, `IndexerProcessorConfig`, `ProcessorStatusSaver` and `get_starting_version()` need some updates. Without these changes, it is difficult to run a live processor at the latest transaction version as well as a backfill processor. ### Updates to Config -[Add an additional field](https://github.com/aptos-labs/aptos-indexer-processors/blob/main/rust/sdk-processor/src/config/indexer_processor_config.rs#L44) on your `IndexerProcessorConfig` for a `BackfillConfig` which contains a single field, `backfill_alias`. The introduction of this field will be used to determine whether the processor is a backfill processor or regular one. +[Add an additional field](https://github.com/aptos-labs/aptos-indexer-processor-example/blob/main/aptos-indexer-processor-example/src/config/indexer_processor_config.rs#L20) on your `IndexerProcessorConfig` for a `BackfillConfig` which contains a single field, `backfill_alias`. The introduction of this field will be used to determine whether the processor is a backfill processor or regular one. ```rust #[derive(Clone, Debug, Deserialize, Serialize)] @@ -53,8 +53,22 @@ pub struct BackfillConfig { } ``` +### Updates to `config.yaml` +Add the `backfill_config` section to `server_config` in your yaml file to set `backfill_alias`. + +```yaml +server_config: + processor_config: + type: "events_processor" + ... + backfill_config: + backfill_alias: "events_processor_backfill" +``` + + ### Backfill Processor Status Table -To avoid write conflicts with the regular `processor_status` table, introduce a `backfill_processor_status_table` that will use `backfill_alias` as the primary key instead of `processor_name`. Expand on this [implementation](https://github.com/aptos-labs/aptos-indexer-processors/blob/main/rust/sdk-processor/src/db/common/models/backfill_processor_status.rs). This model introduces a new state, `BackfillStatus`, which is either `InProgress` or `Complete` which will determine the backfilling restart behavior. +Use a separate table for backfill processor status to avoid write conflicts. This table (`backfill_processor_status_table`) uses `backfill_alias` as the primary key instead of `processor_name` to prevent conflicts with the main `processor_status` table when running head and backfill processors concurrently. Create multiple backfill processors with differing `backfill_alias` and transaction version ranges for a faster backfill. +Expand on this [implementation](https://github.com/aptos-labs/aptos-indexer-processors/blob/main/rust/sdk-processor/src/db/common/models/backfill_processor_status.rs). This model introduces a new state, `BackfillStatus`, which is either `InProgress` or `Complete` which will determine the backfilling restart behavior. ### Updates to ProcessorStatusSaver Expand your `ProcessorStatusSaverEnum` to include a `Backfill` variant that extracts the `backfill_alias` from the `BackfillConfig`, and the `backfill_start_version` `backfill_end_version` from `IndexerProcessorConfig.transaction_stream_config` [like this](https://github.com/aptos-labs/aptos-indexer-processors/blob/main/rust/sdk-processor/src/steps/common/processor_status_saver.rs#L27). Update the corresponding write query to write to the new `backfill_processor_status` table.