feat!: EXPOSED-320 Many-to-many relation with extra columns

- Add more tests (particularly for update) & rename test classes
- Refactor cache to ensure no overlap with wrapped type. Each link entity is now
stored by its target column, source column, and target id (stored in entity)
- Move new entity classes to own file
- Refactor logic for deleting cached entities

documentation-website/Writerside/topics/Deep-Dive-into-DAO.md

@@ -420,7 +420,7 @@ StarWarsFilm.all().first.actors.map { it.roleName }
 ```
 <note>
 If only some additional columns in the intermediate table should be used during batch insert, these can be specified by using
-<code>via()</code> with an argument provided to <code>additionalColumns</code>.
+<code>via()</code> with an argument provided to <code>additionalColumns</code>:
 <code-block>
 class StarWarsFilm(id: EntityID&lt;Int&gt;) : IntEntity(id) {
     companion object : IntEntityClass&lt;StarWarsFilm&gt;(StarWarsFilms)

exposed-dao/src/main/kotlin/org/jetbrains/exposed/dao/EntityCache.kt

@@ -23,7 +23,9 @@ class EntityCache(private val transaction: Transaction) {
     internal val inserts = LinkedHashMap<IdTable<*>, MutableSet<Entity<*>>>()
     private val updates = LinkedHashMap<IdTable<*>, MutableSet<Entity<*>>>()
     internal val referrers = HashMap<Column<*>, MutableMap<EntityID<*>, SizedIterable<*>>>()
-    internal val innerTableLinks by lazy { LinkedHashMap<IdTable<*>, MutableMap<Any, Entity<*>>>() }
+    internal val innerTableLinks by lazy {
+        HashMap<Column<*>, MutableMap<EntityID<*>, MutableSet<InnerTableLinkEntity<*>>>>()
+    }
 
     /**
      * The amount of entities to store in this [EntityCache] per [Entity] class.
@@ -56,11 +58,7 @@ class EntityCache(private val transaction: Transaction) {
             }
         }
 
-    private fun getMap(f: EntityClass<*, *>): MutableMap<Any, Entity<*>> = if (f is InnerTableLinkEntityClass<*, *>) {
-        innerTableLinks.getOrPut(f.table) { LimitedHashMap() }
-    } else {
-        getMap(f.table)
-    }
+    private fun getMap(f: EntityClass<*, *>): MutableMap<Any, Entity<*>> = getMap(f.table)
 
     private fun getMap(table: IdTable<*>): MutableMap<Any, Entity<*>> = data.getOrPut(table) {
         LimitedHashMap()
@@ -104,6 +102,14 @@ class EntityCache(private val transaction: Transaction) {
      */
     fun <ID : Comparable<ID>, T : Entity<ID>> findAll(f: EntityClass<ID, T>): Collection<T> = getMap(f).values as Collection<T>
 
+    internal fun <SID : Comparable<SID>, ID : Comparable<ID>, T : InnerTableLinkEntity<ID>> findInnerTableLink(
+        targetColumn: Column<EntityID<ID>>,
+        targetId: EntityID<ID>,
+        sourceId: EntityID<SID>
+    ): T? {
+        return innerTableLinks[targetColumn]?.get(sourceId)?.firstOrNull { it.id == targetId } as? T
+    }
+
     /** Stores the specified [Entity] in this [EntityCache] using its associated [EntityClass] as the key. */
     fun <ID : Comparable<ID>, T : Entity<ID>> store(f: EntityClass<ID, T>, o: T) {
         getMap(f)[o.id.value] = o
@@ -118,6 +124,14 @@ class EntityCache(private val transaction: Transaction) {
         getMap(o.klass.table)[o.id.value] = o
     }
 
+    internal fun <SID : Comparable<SID>, ID : Comparable<ID>, T : InnerTableLinkEntity<ID>> storeInnerTableLink(
+        targetColumn: Column<EntityID<ID>>,
+        sourceId: EntityID<SID>,
+        targetEntity: T
+    ) {
+        innerTableLinks.getOrPut(targetColumn) { HashMap() }.getOrPut(sourceId) { mutableSetOf() }.add(targetEntity)
+    }
+
     /** Removes the specified [Entity] from this [EntityCache] using its associated [table] as the key. */
     fun <ID : Comparable<ID>, T : Entity<ID>> remove(table: IdTable<ID>, o: T) {
         getMap(table).remove(o.id.value)

exposed-dao/src/main/kotlin/org/jetbrains/exposed/dao/EntityClass.kt

@@ -163,6 +163,13 @@ abstract class EntityClass<ID : Comparable<ID>, out T : Entity<ID>>(
                 with(entity) { col.lookup() }?.let { referrers.remove(it as EntityID<*>) }
             }
         }
+        cache.innerTableLinks.forEach { (_, links) ->
+            links.remove(entity.id)
+
+            links.forEach { (_, targetEntities) ->
+                targetEntities.removeAll { it.wrapped == entity }
+            }
+        }
     }
 
     /** Returns a [SizedIterable] containing all entities with [EntityID] values from the provided [ids] list. */
@@ -207,6 +214,33 @@ abstract class EntityClass<ID : Comparable<ID>, out T : Entity<ID>>(
         wrapRow(it, alias)
     }
 
+    internal fun <SID : Comparable<SID>> wrapLinkRows(
+        rows: SizedIterable<ResultRow>,
+        targetColumn: Column<EntityID<ID>>,
+        sourceColumn: Column<EntityID<SID>>
+    ): SizedIterable<T> = rows mapLazy { wrapLinkRow(it, targetColumn, sourceColumn) }
+
+    private fun <SID : Comparable<SID>> wrapLinkRow(
+        row: ResultRow,
+        targetColumn: Column<EntityID<ID>>,
+        sourceColumn: Column<EntityID<SID>>
+    ): T {
+        val targetId = row[table.id]
+        val sourceId = row[sourceColumn]
+        val transaction = TransactionManager.current()
+        val entity = transaction.entityCache.findInnerTableLink(targetColumn, targetId, sourceId)
+            ?: createInstance(targetId, row).also { new ->
+                new.klass = this
+                new.db = transaction.db
+                warmCache().storeInnerTableLink(targetColumn, sourceId, new as InnerTableLinkEntity<ID>)
+            }
+        if (entity._readValues == null) {
+            entity._readValues = row
+        }
+
+        return entity
+    }
+
     /** Wraps the specified [ResultRow] data into an [Entity] instance. */
     @Suppress("MemberVisibilityCanBePrivate")
     fun wrapRow(row: ResultRow): T {

exposed-dao/src/main/kotlin/org/jetbrains/exposed/dao/EntityLifecycleInterceptor.kt

@@ -96,6 +96,7 @@ class EntityLifecycleInterceptor : GlobalStatementInterceptor {
     override fun beforeRollback(transaction: Transaction) {
         val entityCache = transaction.entityCache
         entityCache.clearReferrersCache()
+        entityCache.innerTableLinks.clear()
         entityCache.data.clear()
         entityCache.inserts.clear()
     }

exposed-dao/src/main/kotlin/org/jetbrains/exposed/dao/InnerTableLink.kt

@@ -90,12 +90,13 @@ class InnerTableLink<SID : Comparable<SID>, Source : Entity<SID>, ID : Comparabl
         val (columns, entityTables) = columnsAndTables
 
         val query = {
-            target.wrapRows(
-                @Suppress("SpreadOperator")
-                entityTables.select(columns)
-                    .where { sourceColumn eq o.id }
-                    .orderBy(*orderByExpressions.toTypedArray())
-            )
+            @Suppress("SpreadOperator")
+            val row = entityTables.select(columns)
+                .where { sourceColumn eq o.id }
+                .orderBy(*orderByExpressions.toTypedArray())
+            (target as? InnerTableLinkEntityClass<ID, *>)
+                ?.wrapLinkRows(row, targetColumn, sourceColumn) as? SizedIterable<Target>
+                ?: target.wrapRows(row)
         }
         return transaction.entityCache.getOrPutReferrers(o.id, sourceColumn, query).also {
             o.storeReferenceInCache(sourceColumn, it)
@@ -120,14 +121,19 @@ class InnerTableLink<SID : Comparable<SID>, Source : Entity<SID>, ID : Comparabl
         val oldValue = getValue(o, unused)
         val existingValues = oldValue.mapIdToAdditionalValues()
         val existingIds = existingValues.keys
-        val additionalColumnsExist = additionalColumns.isNotEmpty()
-        if (additionalColumnsExist) {
-            entityCache.innerTableLinks[target.table]?.remove(o.id.value)
-        }
         entityCache.referrers[sourceColumn]?.remove(o.id)
 
         val targetValues = value.mapIdToAdditionalValues()
         val targetIds = targetValues.keys
+        val additionalColumnsExist = additionalColumns.isNotEmpty()
+        if (additionalColumnsExist) {
+            entityCache.innerTableLinks[targetColumn]?.get(o.id)?.removeAll { cached ->
+                targetValues[cached.id]?.any { (column, targetValue) ->
+                    cached.getInnerTableLinkValue(column) != targetValue
+                } != false
+            }
+        }
+
         executeAsPartOfEntityLifecycle {
             val deleteCondition = if (additionalColumnsExist) {
                 val targetAdditionalValues = targetValues.map { it.value.values.toList() + it.key }
@@ -184,51 +190,3 @@ class InnerTableLink<SID : Comparable<SID>, Source : Entity<SID>, ID : Comparabl
     /** Modifies this reference to sort entities by a column specified in [expression] using ascending order. **/
     infix fun orderBy(expression: Expression<*>) = orderBy(listOf(expression to SortOrder.ASC))
 }
-
-/**
- * Base class for an [Entity] instance identified by a [wrapped] entity comprised of any ID value.
- *
- * Instances of this base class should be used when needing to represent referenced entities in a many-to-many relation
- * from fields defined using `via`, which require additional columns in the intermediate table. These additional
- * columns should be added as constructor properties and the property-column mapping should be defined by
- * [getInnerTableLinkValue].
- *
- * @param WID ID type of the [wrapped] entity instance.
- * @property wrapped The referenced (parent) entity whose unique ID value identifies this [InnerTableLinkEntity] instance.
- * @sample org.jetbrains.exposed.sql.tests.shared.entities.ViaTests.ProjectWithApproval
- */
-abstract class InnerTableLinkEntity<WID : Comparable<WID>>(val wrapped: Entity<WID>) : Entity<WID>(wrapped.id) {
-    /**
-     * Returns the initial column-property mapping for an [InnerTableLinkEntity] instance
-     * before being flushed and inserted into the database.
-     *
-     * @sample org.jetbrains.exposed.sql.tests.shared.entities.ViaTests.ProjectWithApproval
-     */
-    abstract fun getInnerTableLinkValue(column: Column<*>): Any?
-}
-
-/**
- * Base class representing the [EntityClass] that manages [InnerTableLinkEntity] instances and
- * maintains their relation to the provided [table] of the wrapped entity.
- *
- * This should be used, as a companion object to [InnerTableLinkEntity], when needing to represent referenced entities
- * in a many-to-many relation from fields defined using `via`, which require additional columns in the intermediate table.
- * These additional columns will be retrieved as part of a queries [ResultRow] and the column-property mapping to create
- * new instances should be defined by [createInstance].
- *
- * @param WID ID type of the wrapped entity instance.
- * @param E The [InnerTableLinkEntity] type that is managed by this class.
- * @param [table] The [IdTable] object that stores rows mapped to the wrapped entity of this class.
- * @sample org.jetbrains.exposed.sql.tests.shared.entities.ViaTests.ProjectWithApproval
- */
-abstract class InnerTableLinkEntityClass<WID : Comparable<WID>, out E : InnerTableLinkEntity<WID>>(
-    table: IdTable<WID>
-) : EntityClass<WID, E>(table, null, null) {
-    /**
-     * Creates a new [InnerTableLinkEntity] instance by using the provided [row] to both create the wrapped entity
-     * and any additional columns.
-     *
-     * @sample org.jetbrains.exposed.sql.tests.shared.entities.ViaTests.ProjectWithApproval
-     */
-    abstract override fun createInstance(entityId: EntityID<WID>, row: ResultRow?): E
-}

exposed-dao/src/main/kotlin/org/jetbrains/exposed/dao/InnerTableLinkEntity.kt

@@ -0,0 +1,54 @@
+package org.jetbrains.exposed.dao
+
+import org.jetbrains.exposed.dao.id.EntityID
+import org.jetbrains.exposed.dao.id.IdTable
+import org.jetbrains.exposed.sql.Column
+import org.jetbrains.exposed.sql.ResultRow
+
+/**
+ * Base class for an [Entity] instance identified by a [wrapped] entity comprised of any ID value.
+ *
+ * Instances of this base class should be used when needing to represent referenced entities in a many-to-many relation
+ * from fields defined using `via`, which require additional columns in the intermediate table. These additional
+ * columns should be added as constructor properties and the property-column mapping should be defined by
+ * [getInnerTableLinkValue].
+ *
+ * @param WID ID type of the [wrapped] entity instance.
+ * @property wrapped The referenced (parent) entity whose unique ID value identifies this [InnerTableLinkEntity] instance.
+ * @sample org.jetbrains.exposed.sql.tests.shared.entities.ViaTests.ProjectWithApproval
+ */
+abstract class InnerTableLinkEntity<WID : Comparable<WID>>(val wrapped: Entity<WID>) : Entity<WID>(wrapped.id) {
+    /**
+     * Returns the initial column-property mapping for an [InnerTableLinkEntity] instance
+     * before being flushed and inserted into the database.
+     *
+     * @sample org.jetbrains.exposed.sql.tests.shared.entities.ViaTests.ProjectWithApproval
+     */
+    abstract fun getInnerTableLinkValue(column: Column<*>): Any?
+}
+
+/**
+ * Base class representing the [EntityClass] that manages [InnerTableLinkEntity] instances and
+ * maintains their relation to the provided [table] of the wrapped entity.
+ *
+ * This should be used, as a companion object to [InnerTableLinkEntity], when needing to represent referenced entities
+ * in a many-to-many relation from fields defined using `via`, which require additional columns in the intermediate table.
+ * These additional columns will be retrieved as part of a query's [ResultRow] and the column-property mapping to create
+ * new instances should be defined by [createInstance].
+ *
+ * @param WID ID type of the wrapped entity instance.
+ * @param E The [InnerTableLinkEntity] type that is managed by this class.
+ * @param [table] The [IdTable] object that stores rows mapped to the wrapped entity of this class.
+ * @sample org.jetbrains.exposed.sql.tests.shared.entities.ViaTests.ProjectWithApproval
+ */
+abstract class InnerTableLinkEntityClass<WID : Comparable<WID>, out E : InnerTableLinkEntity<WID>>(
+    table: IdTable<WID>
+) : EntityClass<WID, E>(table, null, null) {
+    /**
+     * Creates a new [InnerTableLinkEntity] instance by using the provided [row] to both create the wrapped entity
+     * and any additional columns.
+     *
+     * @sample org.jetbrains.exposed.sql.tests.shared.entities.ViaTests.ProjectWithApproval
+     */
+    abstract override fun createInstance(entityId: EntityID<WID>, row: ResultRow?): E
+}