SONARPY-2080 Move ShannonEntropy to analyzer commons (#340)

Author: ghislainpiot

commons/pom.xml

@@ -106,7 +106,7 @@
                   <!-- Warning: Please consider carefully when increasing the size of this shared library, it might have
                   impact on all our analyzers! -->
                   <minsize>100000</minsize>
-                  <maxsize>120000</maxsize>
+                  <maxsize>122000</maxsize>
                   <files>
                     <file>${project.build.directory}/${project.build.finalName}.jar</file>
                   </files>

commons/src/main/java/org/sonarsource/analyzer/commons/ShannonEntropy.java

@@ -0,0 +1,48 @@
+/*
+ * SonarSource Analyzers Commons
+ * Copyright (C) 2009-2024 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonarsource.analyzer.commons;
+
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import javax.annotation.Nullable;
+
+public class ShannonEntropy {
+  private static final double LOG_2 = Math.log(2.0d);
+
+  private ShannonEntropy() {
+    // utility class
+  }
+
+  public static double calculate(@Nullable String str) {
+    if (str == null || str.isEmpty()) {
+      return 0.0d;
+    }
+    int length = str.length();
+    return str.chars()
+      .boxed()
+      .collect(Collectors.groupingBy(Function.identity(), Collectors.counting()))
+      .values()
+      .stream()
+      .map(Long::doubleValue)
+      .mapToDouble(count -> count / length)
+      .map(frequency -> -frequency * Math.log(frequency))
+      .sum() / LOG_2;
+  }
+}

commons/src/test/java/org/sonarsource/analyzer/commons/ShannonEntropyTest.java

@@ -0,0 +1,54 @@
+/*
+ * SonarSource Analyzers Commons
+ * Copyright (C) 2009-2024 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonarsource.analyzer.commons;
+
+import org.assertj.core.data.Offset;
+import org.junit.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.within;
+
+public class ShannonEntropyTest {
+  private static final Offset<Double> WITHIN_5 = within(0.00001);
+
+  @Test
+  public void calculate_empty() {
+    assertThat(ShannonEntropy.calculate("")).isEqualTo(0.0d);
+    assertThat(ShannonEntropy.calculate(null)).isEqualTo(0.0d);
+  }
+
+  @Test
+  public void calculate_base_2() {
+    assertThat(ShannonEntropy.calculate("ab")).isEqualTo(1.0d);
+  }
+
+  @Test
+  public void calculate_from_sonar_java() {
+    assertThat(ShannonEntropy.calculate("0000000000000000000000000000000000000000")).isEqualTo(0.000000, WITHIN_5);
+    assertThat(ShannonEntropy.calculate("0000000000000000000011111111111111111111")).isEqualTo(1.000000, WITHIN_5);
+    assertThat(ShannonEntropy.calculate("0000000000111111111122222222223333333333")).isEqualTo(2.000000, WITHIN_5);
+    assertThat(ShannonEntropy.calculate("0000011111222223333344444555556666677777")).isEqualTo(3.000000, WITHIN_5);
+    assertThat(ShannonEntropy.calculate("0123456789abcdef0123456789abcdef01234567")).isEqualTo(3.970950, WITHIN_5);
+    assertThat(ShannonEntropy.calculate("0123456789ABCDabcdefghijklmnopqrstuvwxyz")).isEqualTo(5.321928, WITHIN_5);
+    assertThat(ShannonEntropy.calculate("0040878d3579659158d09ad09b6a9849d18e0e22")).isEqualTo(3.587326, WITHIN_5);
+    assertThat(ShannonEntropy.calculate("06c6d5715a1ede6c51fc39ff67fd647f740b656d")).isEqualTo(3.552655, WITHIN_5);
+    assertThat(ShannonEntropy.calculate("qAhEMdXy/MPwEuDlhh7O0AFBuzGvNy7AxpL3sX3q")).isEqualTo(4.684183, WITHIN_5);
+  }
+}