Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

SW-6282 Add is_ad_hoc and observation_type to observations table #2627

Open
wants to merge 6 commits into
base: main
Choose a base branch
from
+85 −0
Open

SW-6282 Add is_ad_hoc and observation_type to observations table #2627

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
906ea34
SW-6282 Add is_ad_hoc and observation_type to observations table
Nov 22, 2024
a74f827
formatting
Nov 22, 2024
42d10c0
Add enums, add missing semicolon
Nov 22, 2024
68cb324
Move enum population and old value migration to the migration script
Nov 26, 2024
e37c14b
Implement is_ad_hoc and observation_type_id into pre-existing observa…
Nov 26, 2024
5558394
Move migration file
Nov 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions jooq/src/main/kotlin/com/terraformation/backend/jooq/Config.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,7 @@ val ENUM_TABLES =
isLocalizable = false),
EnumTable(
"observation_states", listOf("observations\\.state_id"), isLocalizable = false),
EnumTable("observation_types", listOf("observations\\.observation_type_id")),
EnumTable("planting_types"),
EnumTable(
"recorded_plant_statuses",
Expand Down Expand Up @@ -317,6 +318,9 @@ val ID_WRAPPERS =
listOf("monitoring_plot_histories\\.id", ".*\\.monitoring_plot_history_id")),
IdWrapper("MonitoringPlotId", listOf("monitoring_plots\\.id", ".*\\..*_plot_id")),
IdWrapper("ObservationId", listOf("observations\\.id", ".*\\.observation_id")),
IdWrapper(
"ObservationTypeId",
listOf("observation_types\\.id", ".*\\.observation_type_id")),
IdWrapper("ObservedPlotCoordinatesId", listOf("observed_plot_coordinates\\.id")),
IdWrapper("PlantingId", listOf("plantings\\.id")),
IdWrapper("PlantingSeasonId", listOf("planting_seasons\\.id")),
Expand Down
3 changes: 3 additions & 0 deletions src/main/kotlin/com/terraformation/backend/admin/AdminPlantingSitesController.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import com.terraformation.backend.db.default_schema.tables.daos.OrganizationsDao
import com.terraformation.backend.db.tracking.MonitoringPlotId
import com.terraformation.backend.db.tracking.ObservationId
import com.terraformation.backend.db.tracking.ObservationState
import com.terraformation.backend.db.tracking.ObservationType
import com.terraformation.backend.db.tracking.PlantingSeasonId
import com.terraformation.backend.db.tracking.PlantingSiteId
import com.terraformation.backend.db.tracking.PlantingSubzoneId
Expand Down Expand Up @@ -691,6 +692,8 @@ class AdminPlantingSitesController(
NewObservationModel(
endDate = LocalDate.parse(endDate),
id = null,
isAdHoc = false,
observationType = ObservationType.Monitoring,
plantingSiteId = plantingSiteId,
requestedSubzoneIds = requestedSubzoneIds ?: emptySet(),
startDate = LocalDate.parse(startDate),
Expand Down
3 changes: 3 additions & 0 deletions src/main/kotlin/com/terraformation/backend/tracking/api/ObservationsController.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import com.terraformation.backend.db.tracking.ObservationId
import com.terraformation.backend.db.tracking.ObservationPlotPosition
import com.terraformation.backend.db.tracking.ObservationPlotStatus
import com.terraformation.backend.db.tracking.ObservationState
import com.terraformation.backend.db.tracking.ObservationType
import com.terraformation.backend.db.tracking.PlantingSiteId
import com.terraformation.backend.db.tracking.PlantingSubzoneId
import com.terraformation.backend.db.tracking.PlantingZoneId
Expand Down Expand Up @@ -1005,6 +1006,8 @@ data class ScheduleObservationRequestPayload(
NewObservationModel(
endDate = endDate,
id = null,
isAdHoc = false,
observationType = ObservationType.Monitoring,
plantingSiteId = plantingSiteId,
requestedSubzoneIds = requestedSubzoneIds ?: emptySet(),
startDate = startDate,
Expand Down
2 changes: 2 additions & 0 deletions src/main/kotlin/com/terraformation/backend/tracking/db/ObservationStore.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -430,6 +430,8 @@ class ObservationStore(
ObservationsRow(
createdTime = clock.instant(),
endDate = newModel.endDate,
isAdHoc = newModel.isAdHoc,
observationTypeId = newModel.observationType,
plantingSiteId = newModel.plantingSiteId,
startDate = newModel.startDate,
stateId = ObservationState.Upcoming,
Expand Down
5 changes: 5 additions & 0 deletions src/main/kotlin/com/terraformation/backend/tracking/model/ObservationModel.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package com.terraformation.backend.tracking.model

import com.terraformation.backend.db.tracking.ObservationId
import com.terraformation.backend.db.tracking.ObservationState
import com.terraformation.backend.db.tracking.ObservationType
import com.terraformation.backend.db.tracking.PlantingSiteId
import com.terraformation.backend.db.tracking.PlantingSubzoneId
import com.terraformation.backend.db.tracking.tables.references.OBSERVATIONS
Expand All @@ -14,6 +15,8 @@ data class ObservationModel<ID : ObservationId?>(
val completedTime: Instant? = null,
val endDate: LocalDate,
val id: ID,
val isAdHoc: Boolean,
val observationType: ObservationType,
val plantingSiteId: PlantingSiteId,
val requestedSubzoneIds: Set<PlantingSubzoneId> = emptySet(),
val startDate: LocalDate,
Expand Down Expand Up @@ -46,6 +49,8 @@ data class ObservationModel<ID : ObservationId?>(
completedTime = record[OBSERVATIONS.COMPLETED_TIME],
endDate = record[OBSERVATIONS.END_DATE]!!,
id = record[OBSERVATIONS.ID]!!,
isAdHoc = record[OBSERVATIONS.IS_AD_HOC]!!,
observationType = record[OBSERVATIONS.OBSERVATION_TYPE_ID]!!,
plantingSiteId = record[OBSERVATIONS.PLANTING_SITE_ID]!!,
requestedSubzoneIds = record[requestedSubzoneIdsField],
startDate = record[OBSERVATIONS.START_DATE]!!,
Expand Down
28 changes: 28 additions & 0 deletions src/main/resources/db/migration/0300/V321__AdHocPlotObservations.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
CREATE TABLE tracking.observation_types (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL UNIQUE
);

INSERT INTO tracking.observation_types (id, name)
VALUES (1, 'Monitoring'),
(2, 'Biomass Measurements')
ON CONFLICT (id) DO UPDATE SET name = excluded.name;
sgrimm marked this conversation as resolved.
Show resolved Hide resolved

ALTER TABLE tracking.observations ADD COLUMN is_ad_hoc BOOLEAN;

UPDATE tracking.observations
SET is_ad_hoc = false
WHERE is_ad_hoc IS NULL;

ALTER TABLE tracking.observations ALTER COLUMN is_ad_hoc SET NOT NULL;

ALTER TABLE tracking.observations ADD COLUMN observation_type_id INTEGER REFERENCES tracking.observation_types;

UPDATE tracking.observations
SET observation_type_id = 1
WHERE observation_type_id IS NULL;

ALTER TABLE tracking.observations ALTER COLUMN observation_type_id SET NOT NULL;

ALTER TABLE tracking.monitoring_plots ALTER COLUMN planting_subzone_id DROP NOT NULL;
ALTER TABLE tracking.observation_plots ALTER COLUMN monitoring_plot_history_id DROP NOT NULL;
Copy link
Contributor

@sgrimm sgrimm Nov 27, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Dropping the not-null from monitoring plot subzone IDs is definitely needed both for this and for the flexible site editing changes.

I'm nervous about making plot history IDs optional. I think it's likely to force us to write more complicated queries for things like viewing historical ad-hoc observations on a site map since we can no longer count on observation plots being tied to particular map versions via history IDs (meaning we could get the correct map geometry with simple joins on the history IDs) and we'll instead need to calculate the right map to show by, I guess, comparing the observation times against ranges of modification times on the site history.

Also, I think it's plausible we'll allow users to rename or otherwise edit ad-hoc plots after the fact since we're letting users name them initially, which means they would have more than one history row.

Was this just about avoiding storing data that didn't seem necessary, or will it cause problems to keep history for these plots?

2 changes: 2 additions & 0 deletions src/main/resources/db/migration/R__Comments.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -381,6 +381,8 @@ COMMENT ON TABLE tracking.observation_requested_subzones IS 'If an observation s

COMMENT ON TABLE tracking.observation_states IS '(Enum) Where in the observation lifecycle a particular observation is.';

COMMENT ON TABLE tracking.observation_types IS '(Enum) Type of observation, currently only used for ad hoc observations.';

COMMENT ON TABLE tracking.observations IS 'Scheduled observations of planting sites. This table may contain rows describing future observations as well as current and past ones.';
COMMENT ON COLUMN tracking.observations.completed_time IS 'Server-generated date and time the final piece of data for the observation was received.';
COMMENT ON COLUMN tracking.observations.end_date IS 'Last day of the observation. This is typically the last day of the same month as `start_date`.';
Expand Down
2 changes: 2 additions & 0 deletions src/main/resources/i18n/Enums_en.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -306,3 +306,5 @@ tracking.PlantingType.Delivery=Delivery
tracking.PlantingType.ReassignmentFrom=Reassignment From
tracking.PlantingType.ReassignmentTo=Reassignment To
tracking.PlantingType.Undo=Undo
tracking.ObservationType.BiomassMeasurements=Biomass Measurements
tracking.ObservationType.Monitoring=Monitoring
5 changes: 5 additions & 0 deletions src/test/kotlin/com/terraformation/backend/customer/AppNotificationServiceTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ import com.terraformation.backend.db.default_schema.tables.pojos.NotificationsRo
import com.terraformation.backend.db.docprod.VariableType
import com.terraformation.backend.db.nursery.tables.pojos.BatchesRow
import com.terraformation.backend.db.tracking.ObservationState
import com.terraformation.backend.db.tracking.ObservationType
import com.terraformation.backend.device.db.DeviceStore
import com.terraformation.backend.device.event.DeviceUnresponsiveEvent
import com.terraformation.backend.device.event.SensorBoundsAlertTriggeredEvent
Expand Down Expand Up @@ -509,6 +510,8 @@ internal class AppNotificationServiceTest : DatabaseTest(), RunsAsUser {
ExistingObservationModel(
endDate = endDate,
id = inserted.observationId,
isAdHoc = false,
observationType = ObservationType.Monitoring,
plantingSiteId = inserted.plantingSiteId,
startDate = startDate,
state = ObservationState.InProgress)))
Expand Down Expand Up @@ -540,6 +543,8 @@ internal class AppNotificationServiceTest : DatabaseTest(), RunsAsUser {
ExistingObservationModel(
endDate = endDate,
id = inserted.observationId,
isAdHoc = false,
observationType = ObservationType.Monitoring,
plantingSiteId = inserted.plantingSiteId,
startDate = startDate,
state = ObservationState.Upcoming)))
Expand Down
5 changes: 5 additions & 0 deletions src/test/kotlin/com/terraformation/backend/db/DatabaseBackedTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -284,6 +284,7 @@ import com.terraformation.backend.db.tracking.ObservationId
import com.terraformation.backend.db.tracking.ObservationPlotPosition
import com.terraformation.backend.db.tracking.ObservationPlotStatus
import com.terraformation.backend.db.tracking.ObservationState
import com.terraformation.backend.db.tracking.ObservationType
import com.terraformation.backend.db.tracking.ObservedPlotCoordinatesId
import com.terraformation.backend.db.tracking.PlantingId
import com.terraformation.backend.db.tracking.PlantingSeasonId
Expand Down Expand Up @@ -2263,6 +2264,8 @@ abstract class DatabaseBackedTest {
row: ObservationsRow = ObservationsRow(),
createdTime: Instant = Instant.EPOCH,
endDate: LocalDate = row.endDate ?: LocalDate.of(2023, 1, 31),
isAdHoc: Boolean = row.isAdHoc ?: false,
observationType: ObservationType = row.observationTypeId ?: ObservationType.Monitoring,
plantingSiteId: PlantingSiteId = row.plantingSiteId ?: inserted.plantingSiteId,
startDate: LocalDate = row.startDate ?: LocalDate.of(2023, 1, 1),
completedTime: Instant? = row.completedTime,
Expand All @@ -2280,6 +2283,8 @@ abstract class DatabaseBackedTest {
completedTime = completedTime,
createdTime = createdTime,
endDate = endDate,
isAdHoc = isAdHoc,
observationTypeId = observationType,
plantingSiteId = plantingSiteId,
startDate = startDate,
stateId = state,
Expand Down
1 change: 1 addition & 0 deletions src/test/kotlin/com/terraformation/backend/db/SchemaDocsGenerator.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -334,6 +334,7 @@ class SchemaDocsGenerator : DatabaseTest() {
"observation_plots" to setOf(ALL, TRACKING),
"observation_requested_subzones" to setOf(ALL, TRACKING),
"observation_states" to setOf(ALL, TRACKING),
"observation_types" to setOf(ALL, TRACKING),
"observations" to setOf(ALL, TRACKING),
"observed_plot_coordinates" to setOf(ALL, TRACKING),
"observed_plot_species_totals" to setOf(ALL, TRACKING),
Expand Down
7 changes: 7 additions & 0 deletions src/test/kotlin/com/terraformation/backend/email/EmailNotificationServiceTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ import com.terraformation.backend.db.seedbank.AccessionId
import com.terraformation.backend.db.tracking.MonitoringPlotId
import com.terraformation.backend.db.tracking.ObservationId
import com.terraformation.backend.db.tracking.ObservationState
import com.terraformation.backend.db.tracking.ObservationType
import com.terraformation.backend.db.tracking.PlantingSeasonId
import com.terraformation.backend.db.tracking.PlantingSiteId
import com.terraformation.backend.device.db.DeviceStore
Expand Down Expand Up @@ -277,6 +278,8 @@ internal class EmailNotificationServiceTest {
ExistingObservationModel(
endDate = LocalDate.of(2023, 9, 30),
id = ObservationId(1),
isAdHoc = false,
observationType = ObservationType.Monitoring,
plantingSiteId = plantingSite.id,
startDate = LocalDate.of(2023, 9, 1),
state = ObservationState.Upcoming)
Expand Down Expand Up @@ -564,6 +567,8 @@ internal class EmailNotificationServiceTest {
ExistingObservationModel(
endDate = LocalDate.of(2023, 9, 30),
id = ObservationId(1),
isAdHoc = false,
observationType = ObservationType.Monitoring,
plantingSiteId = plantingSite.id,
startDate = LocalDate.of(2023, 9, 1),
state = ObservationState.InProgress))
Expand Down Expand Up @@ -626,6 +631,8 @@ internal class EmailNotificationServiceTest {
ExistingObservationModel(
endDate = LocalDate.of(2023, 10, 31),
id = ObservationId(1),
isAdHoc = false,
observationType = ObservationType.Monitoring,
plantingSiteId = plantingSite.id,
startDate = LocalDate.of(2023, 10, 1),
state = ObservationState.Upcoming))
Expand Down
5 changes: 5 additions & 0 deletions src/test/kotlin/com/terraformation/backend/tracking/ObservationServiceTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import com.terraformation.backend.db.tracking.MonitoringPlotId
import com.terraformation.backend.db.tracking.ObservationId
import com.terraformation.backend.db.tracking.ObservationPlotPosition
import com.terraformation.backend.db.tracking.ObservationState
import com.terraformation.backend.db.tracking.ObservationType
import com.terraformation.backend.db.tracking.PlantingSiteId
import com.terraformation.backend.db.tracking.PlantingSubzoneId
import com.terraformation.backend.db.tracking.RecordedSpeciesCertainty.Known
Expand Down Expand Up @@ -928,6 +929,8 @@ class ObservationServiceTest : DatabaseTest(), RunsAsDatabaseUser {
NewObservationModel(
plantingSiteId = plantingSiteId,
id = null,
isAdHoc = false,
observationType = ObservationType.Monitoring,
startDate = startDate,
endDate = endDate,
state = ObservationState.Upcoming,
Expand Down Expand Up @@ -1740,6 +1743,8 @@ class ObservationServiceTest : DatabaseTest(), RunsAsDatabaseUser {
ExistingObservationModel(
endDate = LocalDate.of(2023, 1, 31),
id = observationId,
isAdHoc = false,
observationType = ObservationType.Monitoring,
plantingSiteId = inserted.plantingSiteId,
startDate = LocalDate.of(2023, 1, 1),
state = ObservationState.InProgress)
Expand Down
13 changes: 13 additions & 0 deletions src/test/kotlin/com/terraformation/backend/tracking/db/ObservationStoreTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import com.terraformation.backend.db.tracking.ObservationId
import com.terraformation.backend.db.tracking.ObservationPlotPosition
import com.terraformation.backend.db.tracking.ObservationPlotStatus
import com.terraformation.backend.db.tracking.ObservationState
import com.terraformation.backend.db.tracking.ObservationType
import com.terraformation.backend.db.tracking.PlantingSiteId
import com.terraformation.backend.db.tracking.PlantingZoneId
import com.terraformation.backend.db.tracking.RecordedPlantStatus.Dead
Expand Down Expand Up @@ -132,6 +133,8 @@ class ObservationStoreTest : DatabaseTest(), RunsAsUser {
ExistingObservationModel(
endDate = endDate1,
id = observationId2,
isAdHoc = false,
observationType = ObservationType.Monitoring,
plantingSiteId = plantingSiteId,
requestedSubzoneIds = setOf(subzoneId),
startDate = startDate1,
Expand All @@ -140,6 +143,8 @@ class ObservationStoreTest : DatabaseTest(), RunsAsUser {
ExistingObservationModel(
endDate = endDate2,
id = observationId1,
isAdHoc = false,
observationType = ObservationType.Monitoring,
plantingSiteId = plantingSiteId,
startDate = startDate2,
state = ObservationState.Upcoming,
Expand Down Expand Up @@ -609,6 +614,8 @@ class ObservationStoreTest : DatabaseTest(), RunsAsUser {
completedTime = Instant.EPOCH,
endDate = LocalDate.of(2020, 1, 31),
id = null,
isAdHoc = false,
observationType = ObservationType.Monitoring,
plantingSiteId = plantingSiteId,
requestedSubzoneIds = setOf(subzoneId1, subzoneId2),
startDate = LocalDate.of(2020, 1, 1),
Expand All @@ -621,6 +628,8 @@ class ObservationStoreTest : DatabaseTest(), RunsAsUser {
createdTime = clock.instant(),
endDate = LocalDate.of(2020, 1, 31),
id = observationId,
isAdHoc = false,
observationTypeId = ObservationType.Monitoring,
plantingSiteId = plantingSiteId,
startDate = LocalDate.of(2020, 1, 1),
stateId = ObservationState.Upcoming,
Expand All @@ -647,6 +656,8 @@ class ObservationStoreTest : DatabaseTest(), RunsAsUser {
NewObservationModel(
endDate = LocalDate.of(2020, 1, 31),
id = null,
isAdHoc = false,
observationType = ObservationType.Monitoring,
plantingSiteId = plantingSiteId,
requestedSubzoneIds = setOf(otherSiteSubzoneId),
startDate = LocalDate.of(2020, 1, 1),
Expand All @@ -664,6 +675,8 @@ class ObservationStoreTest : DatabaseTest(), RunsAsUser {
NewObservationModel(
endDate = LocalDate.EPOCH,
id = null,
isAdHoc = false,
observationType = ObservationType.Monitoring,
plantingSiteId = plantingSiteId,
startDate = LocalDate.EPOCH,
state = ObservationState.Upcoming,
Expand Down