Incrementally rehash for incremental aggregations, use a specialized …

…row set builder for modifications. (#2009)

Integrations/python/deephaven/doc/io/deephaven/engine/table/impl/by/IncrementalChunkedOperatorAggregationStateManager.json

@@ -3,6 +3,7 @@
  "methods": {
   "add": ":param bc: io.deephaven.util.SafeCloseable\n:param rowSequence: io.deephaven.engine.rowset.RowSequence\n:param sources: io.deephaven.engine.table.ColumnSource<?>[]\n:param nextOutputPosition: org.apache.commons.lang3.mutable.MutableInt\n:param outputPositions: io.deephaven.chunk.WritableIntChunk<io.deephaven.engine.rowset.chunkattributes.RowKeys>",
   "addForUpdate": ":param bc: io.deephaven.util.SafeCloseable\n:param rowSequence: io.deephaven.engine.rowset.RowSequence\n:param sources: io.deephaven.engine.table.ColumnSource<?>[]\n:param nextOutputPosition: org.apache.commons.lang3.mutable.MutableInt\n:param outputPositions: io.deephaven.chunk.WritableIntChunk<io.deephaven.engine.rowset.chunkattributes.RowKeys>\n:param reincarnatedPositions: io.deephaven.chunk.WritableIntChunk<io.deephaven.engine.rowset.chunkattributes.RowKeys>",
+  "beginUpdateCycle": "Allow our managers to do a little bit of work at the very start of the update cycle.  We have this method so that\n even if nothing is to be done, we rehash a little bit on each cycle to avoid always rehashing when there is other\n work to be done.",
   "doRehash": ":param bc: io.deephaven.engine.table.impl.by.IncrementalChunkedOperatorAggregationStateManager.BuildContext",
   "findModifications": ":param pc: io.deephaven.util.SafeCloseable\n:param rowSequence: io.deephaven.engine.rowset.RowSequence\n:param sources: io.deephaven.engine.table.ColumnSource<?>[]\n:param outputPositions: io.deephaven.chunk.WritableIntChunk<io.deephaven.engine.rowset.chunkattributes.RowKeys>",
   "findPositionForKey": ":param key: java.lang.Object\n:return: int",

Integrations/python/deephaven/doc/io/deephaven/engine/table/impl/by/IncrementalChunkedOperatorAggregationStateManagerOpenAddressedBase.json

@@ -0,0 +1,16 @@
+{
+ "className": "io.deephaven.engine.table.impl.by.IncrementalChunkedOperatorAggregationStateManagerOpenAddressedBase",
+ "methods": {
+  "add": ":param bc: io.deephaven.util.SafeCloseable\n:param rowSequence: io.deephaven.engine.rowset.RowSequence\n:param sources: io.deephaven.engine.table.ColumnSource<?>[]\n:param nextOutputPosition: org.apache.commons.lang3.mutable.MutableInt\n:param outputPositions: io.deephaven.chunk.WritableIntChunk<io.deephaven.engine.rowset.chunkattributes.RowKeys>",
+  "addForUpdate": ":param bc: io.deephaven.util.SafeCloseable\n:param rowSequence: io.deephaven.engine.rowset.RowSequence\n:param sources: io.deephaven.engine.table.ColumnSource<?>[]\n:param nextOutputPosition: org.apache.commons.lang3.mutable.MutableInt\n:param outputPositions: io.deephaven.chunk.WritableIntChunk<io.deephaven.engine.rowset.chunkattributes.RowKeys>\n:param reincarnatedPositions: io.deephaven.chunk.WritableIntChunk<io.deephaven.engine.rowset.chunkattributes.RowKeys>",
+  "beginUpdateCycle": "Allow our managers to do a little bit of work at the very start of the update cycle.  We have this method so that\n even if nothing is to be done, we rehash a little bit on each cycle to avoid always rehashing when there is other\n work to be done.",
+  "findModifications": ":param pc: io.deephaven.util.SafeCloseable\n:param rowSequence: io.deephaven.engine.rowset.RowSequence\n:param sources: io.deephaven.engine.table.ColumnSource<?>[]\n:param outputPositions: io.deephaven.chunk.WritableIntChunk<io.deephaven.engine.rowset.chunkattributes.RowKeys>",
+  "getKeyHashTableSources": ":return: io.deephaven.engine.table.ColumnSource[]",
+  "makeAggregationStateBuildContext": ":param buildSources: io.deephaven.engine.table.ColumnSource<?>[]\n:param maxSize: long\n:return: io.deephaven.util.SafeCloseable",
+  "onNextChunk": ":param size: int",
+  "remove": ":param pc: io.deephaven.util.SafeCloseable\n:param rowSequence: io.deephaven.engine.rowset.RowSequence\n:param sources: io.deephaven.engine.table.ColumnSource<?>[]\n:param outputPositions: io.deephaven.chunk.WritableIntChunk<io.deephaven.engine.rowset.chunkattributes.RowKeys>\n:param emptiedPositions: io.deephaven.chunk.WritableIntChunk<io.deephaven.engine.rowset.chunkattributes.RowKeys>",
+  "setRowSize": ":param outputPosition: int\n:param size: long"
+ },
+ "path": "io.deephaven.engine.table.impl.by.IncrementalChunkedOperatorAggregationStateManagerOpenAddressedBase",
+ "typeName": "class"
+}

Integrations/python/deephaven/doc/io/deephaven/engine/table/impl/by/IncrementalChunkedOperatorAggregationStateManagerTypedBase.json

@@ -3,6 +3,7 @@
  "methods": {
   "add": ":param bc: io.deephaven.util.SafeCloseable\n:param rowSequence: io.deephaven.engine.rowset.RowSequence\n:param sources: io.deephaven.engine.table.ColumnSource<?>[]\n:param nextOutputPosition: org.apache.commons.lang3.mutable.MutableInt\n:param outputPositions: io.deephaven.chunk.WritableIntChunk<io.deephaven.engine.rowset.chunkattributes.RowKeys>",
   "addForUpdate": ":param bc: io.deephaven.util.SafeCloseable\n:param rowSequence: io.deephaven.engine.rowset.RowSequence\n:param sources: io.deephaven.engine.table.ColumnSource<?>[]\n:param nextOutputPosition: org.apache.commons.lang3.mutable.MutableInt\n:param outputPositions: io.deephaven.chunk.WritableIntChunk<io.deephaven.engine.rowset.chunkattributes.RowKeys>\n:param reincarnatedPositions: io.deephaven.chunk.WritableIntChunk<io.deephaven.engine.rowset.chunkattributes.RowKeys>",
+  "beginUpdateCycle": "Allow our managers to do a little bit of work at the very start of the update cycle.  We have this method so that\n even if nothing is to be done, we rehash a little bit on each cycle to avoid always rehashing when there is other\n work to be done.",
   "findModifications": ":param pc: io.deephaven.util.SafeCloseable\n:param rowSequence: io.deephaven.engine.rowset.RowSequence\n:param sources: io.deephaven.engine.table.ColumnSource<?>[]\n:param outputPositions: io.deephaven.chunk.WritableIntChunk<io.deephaven.engine.rowset.chunkattributes.RowKeys>",
   "getKeyHashTableSources": ":return: io.deephaven.engine.table.ColumnSource[]",
   "makeAggregationStateBuildContext": ":param buildSources: io.deephaven.engine.table.ColumnSource<?>[]\n:param maxSize: long\n:return: io.deephaven.util.SafeCloseable",

Integrations/python/deephaven/doc/io/deephaven/engine/table/impl/by/IncrementalOperatorAggregationStateManager.json

@@ -3,6 +3,7 @@
  "methods": {
   "add": ":param bc: io.deephaven.util.SafeCloseable\n:param rowSequence: io.deephaven.engine.rowset.RowSequence\n:param sources: io.deephaven.engine.table.ColumnSource<?>[]\n:param nextOutputPosition: org.apache.commons.lang3.mutable.MutableInt\n:param outputPositions: io.deephaven.chunk.WritableIntChunk<io.deephaven.engine.rowset.chunkattributes.RowKeys>",
   "addForUpdate": ":param bc: io.deephaven.util.SafeCloseable\n:param rowSequence: io.deephaven.engine.rowset.RowSequence\n:param sources: io.deephaven.engine.table.ColumnSource<?>[]\n:param nextOutputPosition: org.apache.commons.lang3.mutable.MutableInt\n:param outputPositions: io.deephaven.chunk.WritableIntChunk<io.deephaven.engine.rowset.chunkattributes.RowKeys>\n:param reincarnatedPositions: io.deephaven.chunk.WritableIntChunk<io.deephaven.engine.rowset.chunkattributes.RowKeys>",
+  "beginUpdateCycle": "Allow our managers to do a little bit of work at the very start of the update cycle.  We have this method so that\n even if nothing is to be done, we rehash a little bit on each cycle to avoid always rehashing when there is other\n work to be done.",
   "findModifications": ":param pc: io.deephaven.util.SafeCloseable\n:param rowSequence: io.deephaven.engine.rowset.RowSequence\n:param sources: io.deephaven.engine.table.ColumnSource<?>[]\n:param outputPositions: io.deephaven.chunk.WritableIntChunk<io.deephaven.engine.rowset.chunkattributes.RowKeys>",
   "findPositionForKey": ":param key: java.lang.Object\n:return: int",
   "getKeyHashTableSources": ":return: io.deephaven.engine.table.ColumnSource[]",

Integrations/python/deephaven/doc/io/deephaven/engine/table/impl/by/OperatorAggregationStateManagerOpenAddressedAlternateBase.json

@@ -0,0 +1,11 @@
+{
+ "className": "io.deephaven.engine.table.impl.by.OperatorAggregationStateManagerOpenAddressedAlternateBase",
+ "methods": {
+  "doRehash": ":param fullRehash: (boolean) - should we rehash the entire table (if false, we rehash incrementally)\n:param rehashCredits: (org.apache.commons.lang3.mutable.MutableInt) - the number of entries this operation has rehashed (input/output)\n:param nextChunkSize: (int) - the size of the chunk we are processing\n:return: (boolean) true if a front migration is required",
+  "findPositionForKey": ":param key: java.lang.Object\n:return: int",
+  "makeProbeContext": ":param buildSources: io.deephaven.engine.table.ColumnSource<?>[]\n:param maxSize: long\n:return: io.deephaven.engine.table.impl.by.OperatorAggregationStateManagerOpenAddressedAlternateBase.ProbeContext",
+  "rehashRequired": ":param nextChunkSize: int\n:return: boolean"
+ },
+ "path": "io.deephaven.engine.table.impl.by.OperatorAggregationStateManagerOpenAddressedAlternateBase",
+ "typeName": "class"
+}

Integrations/python/deephaven/doc/io/deephaven/engine/table/impl/by/OperatorAggregationStateManagerOpenAddressedAlternateBase/BuildContext.json

@@ -0,0 +1,8 @@
+{
+ "className": "io.deephaven.engine.table.impl.by.OperatorAggregationStateManagerOpenAddressedAlternateBase$BuildContext",
+ "methods": {
+  "close": "Release any resources associated with this context. The context should not be used afterwards."
+ },
+ "path": "io.deephaven.engine.table.impl.by.OperatorAggregationStateManagerOpenAddressedAlternateBase.BuildContext",
+ "typeName": "class"
+}

Integrations/python/deephaven/doc/io/deephaven/engine/table/impl/by/OperatorAggregationStateManagerOpenAddressedAlternateBase/BuildHandler.json

@@ -0,0 +1,8 @@
+{
+ "className": "io.deephaven.engine.table.impl.by.OperatorAggregationStateManagerOpenAddressedAlternateBase$BuildHandler",
+ "methods": {
+  "doBuild": ":param chunkOk: io.deephaven.engine.rowset.RowSequence\n:param sourceKeyChunks: io.deephaven.chunk.Chunk<io.deephaven.chunk.attributes.Values>[]"
+ },
+ "path": "io.deephaven.engine.table.impl.by.OperatorAggregationStateManagerOpenAddressedAlternateBase.BuildHandler",
+ "typeName": "interface"
+}

Integrations/python/deephaven/doc/io/deephaven/engine/table/impl/by/OperatorAggregationStateManagerOpenAddressedAlternateBase/ProbeContext.json

@@ -0,0 +1,8 @@
+{
+ "className": "io.deephaven.engine.table.impl.by.OperatorAggregationStateManagerOpenAddressedAlternateBase$ProbeContext",
+ "methods": {
+  "close": "Release any resources associated with this context. The context should not be used afterwards."
+ },
+ "path": "io.deephaven.engine.table.impl.by.OperatorAggregationStateManagerOpenAddressedAlternateBase.ProbeContext",
+ "typeName": "class"
+}

Integrations/python/deephaven/doc/io/deephaven/engine/table/impl/by/OperatorAggregationStateManagerOpenAddressedAlternateBase/ProbeHandler.json

@@ -0,0 +1,8 @@
+{
+ "className": "io.deephaven.engine.table.impl.by.OperatorAggregationStateManagerOpenAddressedAlternateBase$ProbeHandler",
+ "methods": {
+  "doProbe": ":param chunkOk: io.deephaven.engine.rowset.RowSequence\n:param sourceKeyChunks: io.deephaven.chunk.Chunk<io.deephaven.chunk.attributes.Values>[]"
+ },
+ "path": "io.deephaven.engine.table.impl.by.OperatorAggregationStateManagerOpenAddressedAlternateBase.ProbeHandler",
+ "typeName": "interface"
+}

Integrations/python/deephaven/doc/io/deephaven/engine/table/impl/by/alternatingcolumnsource/AlternatingColumnSource.json

@@ -0,0 +1,41 @@
+{
+ "className": "io.deephaven.engine.table.impl.by.alternatingcolumnsource.AlternatingColumnSource",
+ "methods": {
+  "allowsReinterpret": "Test if a reinterpret call will succeed.\n\nNote: Java generics information - <ALTERNATE_DATA_TYPE>\n\n:param alternateDataType: (java.lang.Class<ALTERNATE_DATA_TYPE>) - The alternative type to consider\n:return: (boolean) If a reinterpret on this column source with the supplied alternateDataType will succeed.",
+  "fillChunk": "Populates the given destination chunk with data corresponding to the keys from the given RowSequence.\n\n:param context: (io.deephaven.engine.table.ChunkSource.FillContext) - A context containing all mutable/state related data used in retrieving the Chunk.\n:param destination: (io.deephaven.chunk.WritableChunk<? superio.deephaven.chunk.attributes.Values>) - The chunk to be populated according to rowSequence. No assumptions shall be made about\n        the size of the chunk shall be made. The chunk will be populated from position [0,rowSequence.size()).\n:param rowSequence: (io.deephaven.engine.rowset.RowSequence) - An RowSequence representing the keys to be fetched",
+  "fillChunkUnordered": "Populates a contiguous portion of the given destination chunk with data corresponding to the keys from the given\n LongChunk.\n\n:param context: (io.deephaven.engine.table.ChunkSource.FillContext) - A context containing all mutable/state related data used in retrieving the Chunk.\n:param dest: (io.deephaven.chunk.WritableChunk<? superio.deephaven.chunk.attributes.Values>) - The chunk to be populated according to keys\n:param keys: (io.deephaven.chunk.LongChunk<? extendsio.deephaven.engine.rowset.chunkattributes.RowKeys>) - A chunk of individual, not assumed to be ordered keys to be fetched",
+  "fillPrevChunk": "Populates the given destination chunk with data corresponding to the keys from the given RowSequence.\n\n:param context: (io.deephaven.engine.table.ChunkSource.FillContext) - A context containing all mutable/state related data used in retrieving the Chunk.\n:param destination: (io.deephaven.chunk.WritableChunk<? superio.deephaven.chunk.attributes.Values>) - The chunk to be populated according to rowSequence. No assumptions shall be made\n        about the size of the chunk shall be made. The chunk will be populated from position\n        [0,rowSequence.size()).\n:param rowSequence: (io.deephaven.engine.rowset.RowSequence) - An RowSequence representing the keys to be fetched",
+  "fillPrevChunkUnordered": "Populates a contiguous portion of the given destination chunk with prev data corresponding to the keys from the\n given LongChunk.\n\n:param context: (io.deephaven.engine.table.ChunkSource.FillContext) - A context containing all mutable/state related data used in retrieving the Chunk.\n:param dest: (io.deephaven.chunk.WritableChunk<? superio.deephaven.chunk.attributes.Values>) - The chunk to be populated according to keys\n:param keys: (io.deephaven.chunk.LongChunk<? extendsio.deephaven.engine.rowset.chunkattributes.RowKeys>) - A chunk of individual, not assumed to be ordered keys to be fetched",
+  "get": ":param index: long\n:return: AlternatingColumnSource.DATA_TYPE",
+  "getBoolean": ":param index: long\n:return: java.lang.Boolean",
+  "getByte": ":param index: long\n:return: byte",
+  "getChar": ":param index: long\n:return: char",
+  "getChunk": "Returns a chunk of data corresponding to the keys from the given RowSequence.\n\n:param context: (io.deephaven.engine.table.ChunkSource.GetContext) - A context containing all mutable/state related data used in retrieving the Chunk. In particular,\n        the Context may be used to provide a Chunk data pool\n:param rowSequence: (io.deephaven.engine.rowset.RowSequence) - An RowSequence representing the keys to be fetched\n:return: (io.deephaven.chunk.Chunk<? extends io.deephaven.chunk.attributes.Values>) A chunk of data corresponding to the keys from the given RowSequence",
+  "getDouble": ":param index: long\n:return: double",
+  "getFloat": ":param index: long\n:return: float",
+  "getInt": ":param index: long\n:return: int",
+  "getLong": ":param index: long\n:return: long",
+  "getPrev": ":param index: long\n:return: AlternatingColumnSource.DATA_TYPE",
+  "getPrevBoolean": ":param index: long\n:return: java.lang.Boolean",
+  "getPrevByte": ":param index: long\n:return: byte",
+  "getPrevChar": ":param index: long\n:return: char",
+  "getPrevChunk": "Returns a chunk of previous data corresponding to the keys from the given RowSequence.\n\n:param context: (io.deephaven.engine.table.ChunkSource.GetContext) - A context containing all mutable/state related data used in retrieving the Chunk. In\n        particular, the Context may be used to provide a Chunk data pool\n:param rowSequence: (io.deephaven.engine.rowset.RowSequence) - An RowSequence representing the keys to be fetched\n:return: (io.deephaven.chunk.Chunk<? extends io.deephaven.chunk.attributes.Values>) A chunk of data corresponding to the keys from the given RowSequence",
+  "getPrevDouble": ":param index: long\n:return: double",
+  "getPrevFloat": ":param index: long\n:return: float",
+  "getPrevInt": ":param index: long\n:return: int",
+  "getPrevLong": ":param index: long\n:return: long",
+  "getPrevShort": ":param index: long\n:return: short",
+  "getShort": ":param index: long\n:return: short",
+  "innerLocation": ":param hashSlot: long\n:return: int",
+  "isAlternate": ":param index: long\n:return: boolean",
+  "isImmutable": "Determine if this column source is immutable, meaning that the values at a given row key never change.\n\n:return: (boolean) true if the values at a given row key of the column source never change, false otherwise",
+  "makeFillContext": "Allocate a new ChunkSource.FillContext for filling chunks from this FillContextMaker, typically a\n ChunkSource.\n\n:param chunkCapacity: (int) - The maximum size of any WritableChunk that will be filled with this context\n:param sharedContext: (io.deephaven.engine.table.SharedContext) - Shared store of intermediate results.\n:return: (io.deephaven.engine.table.ChunkSource.FillContext) A context for use with fill operations",
+  "makeGetContext": "Allocate a new ChunkSource.GetContext for retrieving chunks from this GetContextMaker, typically\n a ChunkSource.\n\n:param chunkCapacity: (int) - The maximum size required for any WritableChunk allocated as part of the result.\n:param sharedContext: (io.deephaven.engine.table.SharedContext) - Shared store of intermediate results.\n:return: (io.deephaven.engine.table.ChunkSource.GetContext) A context for use with get operations",
+  "providesFillUnordered": "Returns true if this column source can efficiently provide an unordered fill.\n\n If this method returns false, then fillChunkUnordered and fillPrevChunkUnordered may throw an\n UnsupportedOperationException.\n\n:return: (boolean) if this column source can provide an unordered fill",
+  "setSources": ":param mainSource: io.deephaven.engine.table.ColumnSource<AlternatingColumnSource.DATA_TYPE>\n:param alternateSource: io.deephaven.engine.table.ColumnSource<AlternatingColumnSource.DATA_TYPE>",
+  "startTrackingPrevValues": "ColumnSource implementations that track previous values have the option to not actually start tracking previous\n values until this method is called. This is an option, not an obligation: some simple ColumnSource\n implementations (like TSingleValueSource for various T) always track previous values; other implementations (like\n PrevColumnSource) never do; some (like TArrayColumnSource) only start tracking once this method is called.\n\n An immutable column source can not have distinct prev values; therefore it is implemented as a no-op."
+ },
+ "path": "io.deephaven.engine.table.impl.by.alternatingcolumnsource.AlternatingColumnSource",
+ "text": "ColumnSource implementation that delegates to the main and alternate sources for our incremental open\n addressed hash table key columns that swap back and forth between a \"main\" and \"alternate\" source. Note that the main\n and alternate swap back and forth, from the perspective of this column source the main source is addressed by zero;\n and the alternate source is addressed starting at ALTERNATE_SWITCH_MASK. Neither source may have addresses\n greater than ALTERNATE_INNER_MASK.",
+ "typeName": "class"
+}

Integrations/python/deephaven/doc/io/deephaven/engine/table/impl/by/alternatingcolumnsource/ByteAlternatingColumnSourceUnorderedMergeKernel.json

@@ -0,0 +1,8 @@
+{
+ "className": "io.deephaven.engine.table.impl.by.alternatingcolumnsource.ByteAlternatingColumnSourceUnorderedMergeKernel",
+ "methods": {
+  "mergeContext": ":param dest: io.deephaven.chunk.WritableChunk<? superio.deephaven.chunk.attributes.Values>\n:param outerKeys: io.deephaven.chunk.LongChunk<? extendsio.deephaven.engine.rowset.chunkattributes.RowKeys>\n:param src: io.deephaven.chunk.Chunk<? superio.deephaven.chunk.attributes.Values>\n:param alternatePosition: int"
+ },
+ "path": "io.deephaven.engine.table.impl.by.alternatingcolumnsource.ByteAlternatingColumnSourceUnorderedMergeKernel",
+ "typeName": "class"
+}

Integrations/python/deephaven/doc/io/deephaven/engine/table/impl/by/alternatingcolumnsource/CharAlternatingColumnSourceUnorderedMergeKernel.json

@@ -0,0 +1,8 @@
+{
+ "className": "io.deephaven.engine.table.impl.by.alternatingcolumnsource.CharAlternatingColumnSourceUnorderedMergeKernel",
+ "methods": {
+  "mergeContext": ":param dest: io.deephaven.chunk.WritableChunk<? superio.deephaven.chunk.attributes.Values>\n:param outerKeys: io.deephaven.chunk.LongChunk<? extendsio.deephaven.engine.rowset.chunkattributes.RowKeys>\n:param src: io.deephaven.chunk.Chunk<? superio.deephaven.chunk.attributes.Values>\n:param alternatePosition: int"
+ },
+ "path": "io.deephaven.engine.table.impl.by.alternatingcolumnsource.CharAlternatingColumnSourceUnorderedMergeKernel",
+ "typeName": "class"
+}