IGNITE-24098 Merged master.

Author: vladnovoren

modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/ExpiredEntriesIntegrationTest.java

@@ -23,6 +23,9 @@
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.cache.query.annotations.QuerySqlField;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.metric.IoStatisticsHolder;
+import org.apache.ignite.internal.processors.query.calcite.QueryChecker;
+import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.junit.Test;
 
@@ -35,23 +38,29 @@
  */
 public class ExpiredEntriesIntegrationTest extends AbstractBasicIntegrationTest {
     /** */
-    @Test
-    public void testExpiration() throws Exception {
+    @Override protected void beforeTest() throws Exception {
         CacheConfiguration<Integer, Developer> cacheCfg = new CacheConfiguration<Integer, Developer>()
             .setIndexedTypes(Integer.class, Developer.class)
             .setExpiryPolicyFactory(CreatedExpiryPolicy.factoryOf(new Duration(TimeUnit.MILLISECONDS, 1)));
 
-        IgniteCache<Integer, Developer> cache1 = client.getOrCreateCache(new CacheConfiguration<>(cacheCfg)
+        client.createCache(new CacheConfiguration<>(cacheCfg)
             .setName("CACHE1")
             .setEagerTtl(false)
         );
 
-        IgniteCache<Integer, Developer> cache2 = client.getOrCreateCache(new CacheConfiguration<>(cacheCfg)
+        client.createCache(new CacheConfiguration<>(cacheCfg)
             .setName("CACHE2")
             .setEagerTtl(true)
         );
 
         awaitPartitionMapExchange();
+    }
+
+    /** */
+    @Test
+    public void testExpiration() throws Exception {
+        IgniteCache<Integer, Developer> cache1 = client.cache("CACHE1");
+        IgniteCache<Integer, Developer> cache2 = client.cache("CACHE2");
 
         for (int i = 0; i < 100; i++) {
             cache1.put(i, new Developer("dev" + i, i));
@@ -71,6 +80,30 @@ public void testExpiration() throws Exception {
         checkExpiration("CACHE2", true);
     }
 
+    /** Validate that check of expiry policy doesn't require additional page read. */
+    @Test
+    public void testIndexScanReadPageOnce() {
+        for (String cacheName: F.asList("CACHE1", "CACHE2")) {
+            IgniteCache<Integer, Developer> cache = client.cache(cacheName);
+
+            ExpiryPolicy expPlc = new CreatedExpiryPolicy(new Duration(TimeUnit.DAYS, 1));
+
+            int key = primaryKey(grid(0).cache(cacheName));
+
+            cache.withExpiryPolicy(expPlc).put(key, new Developer("dev0", 0));
+
+            IoStatisticsHolder statHld = grid(0).cachex(cacheName).context().group().statisticsHolderData();
+
+            long before = statHld.logicalReads();
+
+            assertQuery("SELECT /*+ FORCE_INDEX(DEVELOPER_DEPID_IDX) */ name, depId FROM " + cacheName + ".DEVELOPER where depId=0")
+                .matches(QueryChecker.containsIndexScan(cacheName, "DEVELOPER", "DEVELOPER_DEPID_IDX"))
+                .check();
+
+            assertEquals(1, statHld.logicalReads() - before);
+        }
+    }
+
     /** */
     private void checkExpiration(String schema, boolean eagerTtl) {
         assertQuery("SELECT depId, name FROM " + schema + ".DEVELOPER WHERE name IS NOT NULL")

modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/CliIgniteClientInvoker.java

@@ -17,9 +17,11 @@
 
 package org.apache.ignite.internal.commandline;
 
+import java.util.Collection;
 import java.util.function.Consumer;
 import org.apache.ignite.Ignition;
 import org.apache.ignite.client.IgniteClient;
+import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.ClientConfiguration;
 import org.apache.ignite.internal.client.GridClientNode;
 import org.apache.ignite.internal.client.GridClientNodeStateBeforeStart;
@@ -34,6 +36,7 @@
 import org.jetbrains.annotations.Nullable;
 
 import static org.apache.ignite.internal.processors.odbc.ClientListenerNioListener.MANAGEMENT_CLIENT_ATTR;
+import static org.apache.ignite.internal.processors.odbc.ClientListenerProcessor.CLIENT_LISTENER_PORT;
 
 /**
  * Adapter of new management API command for {@code control.sh} execution flow.
@@ -54,7 +57,16 @@ public CliIgniteClientInvoker(Command<A, ?> cmd, A arg, ClientConfiguration cfg)
 
     /** {@inheritDoc} */
     @Override protected GridClientNode defaultNode() {
-        return CommandUtils.clusterToClientNode(igniteClient().cluster().forOldest().node());
+        String[] addr = cfg.getAddresses()[0].split(":");
+
+        String host = addr[0];
+        String port = addr[1];
+
+        Collection<ClusterNode> nodes = igniteClient().cluster().nodes();
+
+        return CommandUtils.clusterToClientNode(F.find(nodes, U.oldest(nodes, null), node ->
+            (node.hostNames().contains(host) || node.addresses().contains(host))
+                && port.equals(node.attribute(CLIENT_LISTENER_PORT).toString())));
     }
 
     /** {@inheritDoc} */

modules/control-utility/src/test/java/org/apache/ignite/testsuites/IgniteControlUtilityTestSuite2.java

@@ -34,6 +34,7 @@
 import org.apache.ignite.util.GridCommandHandlerPropertiesTest;
 import org.apache.ignite.util.GridCommandHandlerScheduleIndexRebuildTest;
 import org.apache.ignite.util.GridCommandHandlerTracingConfigurationTest;
+import org.apache.ignite.util.IdleVerifyDumpTest;
 import org.apache.ignite.util.MetricCommandTest;
 import org.apache.ignite.util.PerformanceStatisticsCommandTest;
 import org.apache.ignite.util.SystemViewCommandTest;
@@ -72,7 +73,9 @@
     CdcCommandTest.class,
     CdcResendCommandTest.class,
 
-    SecurityCommandHandlerPermissionsTest.class
+    SecurityCommandHandlerPermissionsTest.class,
+
+    IdleVerifyDumpTest.class
 })
 public class IgniteControlUtilityTestSuite2 {
 }

modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerClusterByClassTest.java

@@ -752,7 +752,7 @@ public void testCacheIdleVerifyDump() throws Exception {
             String dumpWithZeros = new String(Files.readAllBytes(Paths.get(fileNameMatcher.group(1))));
 
             assertContains(log, dumpWithZeros, "The check procedure has finished, found " + parts + " partitions");
-            assertContains(log, dumpWithZeros, "Partition: PartitionKeyV2 [grpId=1544803905, grpName=default, partId=0]");
+            assertContains(log, dumpWithZeros, "Partition: PartitionKey [grpId=1544803905, grpName=default, partId=0]");
             assertContains(log, dumpWithZeros, "updateCntr=0, partitionState=OWNING, size=0, partHash=0");
             assertContains(log, dumpWithZeros, "no conflicts have been found");
             assertCompactFooterStat(dumpWithZeros, 0, 0, 0, keysCnt);
@@ -769,7 +769,7 @@ public void testCacheIdleVerifyDump() throws Exception {
 
             assertContains(log, dumpWithoutZeros, "The check procedure has finished, found " + keysCnt + " partitions");
             assertContains(log, dumpWithoutZeros, (parts - keysCnt) + " partitions was skipped");
-            assertContains(log, dumpWithoutZeros, "Partition: PartitionKeyV2 [grpId=1544803905, grpName=default, partId=");
+            assertContains(log, dumpWithoutZeros, "Partition: PartitionKey [grpId=1544803905, grpName=default, partId=");
 
             assertNotContains(log, dumpWithoutZeros, "updateCntr=0, partitionState=OWNING, size=0, partHash=0");
 
@@ -988,7 +988,7 @@ private void testCacheIdleVerifyMultipleCacheFilterOptionsCommon(
      */
     private void assertSort(int expectedPartsCount, String output) {
         Pattern partIdPattern = Pattern.compile(".*partId=([0-9]*)");
-        Pattern primaryPattern = Pattern.compile("Partition instances: \\[PartitionHashRecordV2 \\[isPrimary=true");
+        Pattern primaryPattern = Pattern.compile("Partition instances: \\[PartitionHashRecord \\[isPrimary=true");
 
         Matcher partIdMatcher = partIdPattern.matcher(output);
         Matcher primaryMatcher = primaryPattern.matcher(output);
@@ -1250,7 +1250,7 @@ public void testCacheIdleVerifyDumpExcludedCaches() throws Exception {
     /**
      * @return Build matcher for dump file name.
      */
-    @NotNull private Matcher dumpFileNameMatcher() {
+    @NotNull static Matcher dumpFileNameMatcher() {
         Pattern fileNamePattern = Pattern.compile(".*" + IdleVerifyDumpTask.class.getSimpleName()
             + " successfully written output to '(.*)'");
         return fileNamePattern.matcher(testOut.toString());

modules/control-utility/src/test/java/org/apache/ignite/util/IdleVerifyDumpTest.java

@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.util;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.regex.Matcher;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.junit.Test;
+
+import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK;
+import static org.apache.ignite.testframework.GridTestUtils.assertContains;
+import static org.apache.ignite.util.GridCommandHandlerClusterByClassTest.dumpFileNameMatcher;
+
+/** */
+public class IdleVerifyDumpTest extends GridCommandHandlerClusterByClassAbstractTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        return super.getConfiguration(igniteInstanceName)
+            .setWorkDirectory(nodeWorkDirectory(igniteInstanceName));
+    }
+
+    /**
+     * Test ensuring that idle verify dump output file is created exactly
+     * on server node specified via the --host parameter.
+     */
+    @Test
+    public void testDumpResultMatchesConnection() throws Exception {
+        injectTestSystemOut();
+
+        client.createCache(DEFAULT_CACHE_NAME).put(1, 1);
+
+        checkDump(0);
+
+        checkDump(1);
+    }
+
+    /** */
+    private void checkDump(int nodeIdx) throws Exception {
+        assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", "--port", connectorPort(grid(nodeIdx))));
+
+        Matcher fileNameMatcher = dumpFileNameMatcher();
+
+        assertTrue(fileNameMatcher.find());
+
+        Path dumpFileName = Paths.get(fileNameMatcher.group(1));
+
+        String dumpRes = new String(Files.readAllBytes(dumpFileName));
+
+        assertContains(log, dumpRes, "The check procedure has finished, no conflicts have been found.");
+
+        assertContains(log, dumpFileName.toString(), nodeWorkDirectory(getTestIgniteInstanceName(nodeIdx)));
+    }
+
+    /** */
+    private String nodeWorkDirectory(String igniteInstanceName) throws IgniteCheckedException {
+        return new File(U.defaultWorkDirectory(), igniteInstanceName).getAbsolutePath();
+    }
+}

modules/core/src/main/java/org/apache/ignite/internal/cache/query/index/sorted/inline/InlineTreeFilterClosure.java

@@ -25,6 +25,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusIO;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.spi.indexing.IndexingQueryCacheFilter;
+import org.jetbrains.annotations.Nullable;
 
 import static org.apache.ignite.internal.pagemem.PageIdUtils.pageId;
 
@@ -38,6 +39,9 @@ public class InlineTreeFilterClosure implements BPlusTree.TreeRowClosure<IndexRo
     /** */
     private final BPlusTree.TreeRowClosure<IndexRow, IndexRow> rowFilter;
 
+    /** Last index row analyzed by {@link #rowFilter}. */
+    private @Nullable IndexRow lastRow;
+
     /** Constructor. */
     public InlineTreeFilterClosure(IndexingQueryCacheFilter cacheFilter,
         BPlusTree.TreeRowClosure<IndexRow, IndexRow> rowFilter) {
@@ -53,15 +57,22 @@ public InlineTreeFilterClosure(IndexingQueryCacheFilter cacheFilter,
 
         boolean val = cacheFilter == null || applyFilter((InlineIO)io, pageAddr, idx);
 
-        if (!val)
-            return false;
-
-        if (rowFilter != null)
+        if (val && rowFilter != null) {
             val = rowFilter.apply(tree, io, pageAddr, idx);
 
+            lastRow = rowFilter.lastRow();
+        }
+        else
+            lastRow = null;
+
         return val;
     }
 
+    /** {@inheritDoc} */
+    @Override public @Nullable IndexRow lastRow() {
+        return lastRow;
+    }
+
     /**
      * @param io Row IO.
      * @param pageAddr Page address.

modules/core/src/main/java/org/apache/ignite/internal/management/cache/IdleVerifyDumpResult.java

@@ -21,7 +21,7 @@
 import java.io.ObjectOutput;
 import java.util.List;
 import java.util.Map;
-import org.apache.ignite.internal.processors.cache.verify.PartitionHashRecordV2;
+import org.apache.ignite.internal.processors.cache.verify.PartitionHashRecord;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.visor.VisorDataTransferObject;
@@ -34,12 +34,12 @@ public class IdleVerifyDumpResult extends VisorDataTransferObject {
     private static final long serialVersionUID = 0L;
 
     /** Cluster hashes. */
-    private Map<PartitionKeyV2, List<PartitionHashRecordV2>> clusterHashes;
+    private Map<PartitionKey, List<PartitionHashRecord>> clusterHashes;
 
     /**
      * @param clusterHashes Cluster hashes.
      */
-    public IdleVerifyDumpResult(Map<PartitionKeyV2, List<PartitionHashRecordV2>> clusterHashes) {
+    public IdleVerifyDumpResult(Map<PartitionKey, List<PartitionHashRecord>> clusterHashes) {
         this.clusterHashes = clusterHashes;
     }
 
@@ -63,7 +63,7 @@ public IdleVerifyDumpResult() {
     /**
      * @return Cluster hashes.
      */
-    public Map<PartitionKeyV2, List<PartitionHashRecordV2>> clusterHashes() {
+    public Map<PartitionKey, List<PartitionHashRecord>> clusterHashes() {
         return clusterHashes;
     }