Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix DB unit tests #3186

Merged
merged 24 commits into from
Dec 18, 2024
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
86127f9
add docker compose that allows all services to run locally
fhanik Dec 9, 2024
fd5adcc
Fix flakiness in a test that does not yield time change on a fast system
fhanik Dec 9, 2024
ee70778
tests: fix unit tests to support all three databases
fhanik Dec 10, 2024
3a8d0c6
tests: use java initializer in TableAndColumnNormalizationTest
Kehrlann Dec 4, 2024
b81e0cc
tests: introduce @EnabledIfProfile / @DisabledIfProfile
Kehrlann Dec 9, 2024
80de45e
tests: fix hsql-specific JdbcPagingListTests
Kehrlann Dec 9, 2024
0596320
tests: fix JdbcScimUserProvisioningTests.retrieveByScimFilter_Include…
Kehrlann Dec 10, 2024
2ca7f14
tests: fix JdbcClientMetadataProvisioningTest.createdByPadsTo36Chars …
Kehrlann Dec 10, 2024
2ae2439
Remove hard coded database values from the MockMvc integration tests
fhanik Dec 10, 2024
c0162f5
Downgrade MySQL to avoid padding issues
fhanik Dec 10, 2024
97f475a
Remove database connections leaks (1 prod, many in tests)
fhanik Dec 11, 2024
5d33664
Change table columns `users.id` to varchar from char
fhanik Dec 11, 2024
0eb383a
Ensure that the table `groups` is properly escaped since groups is a …
fhanik Dec 11, 2024
49f7a30
add TODO in ScimUserEndpointsAliasMockMvcTests.AliasFeatureEnabled
fhanik Dec 11, 2024
4b02a1b
No more CHAR datatypes, only varchar
fhanik Dec 11, 2024
eece476
Fix mistake on column null handling
fhanik Dec 12, 2024
fea1a59
tests: disable BootstrapTests in mysql and postgresql profiles
Kehrlann Dec 13, 2024
03b544e
tests: remove duplicate test
Kehrlann Dec 16, 2024
34656be
tests: ensure mysql is on the same timezone as the host machine when …
Kehrlann Dec 16, 2024
9436ed4
Fix connection leak from Flyway
fhanik Dec 16, 2024
428b189
tests: database data volumes in docker-compose.yml uses tmpfs
Kehrlann Dec 17, 2024
de56eab
tests: re-enable createdByPadsTo36Chars excpet for MySQL
Kehrlann Dec 17, 2024
8738e29
docs: add docs/testing.md
Kehrlann Dec 17, 2024
345c7c1
tests: rework JdbcScimUserProvisioningTests.retrieveByScimFilter_Incl…
Kehrlann Dec 17, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 38 additions & 0 deletions scripts/docker-compose.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
name: uaa

services:
postgres:
image: "postgres:17.2"
strehle marked this conversation as resolved.
Show resolved Hide resolved
ports:
- 5432:5432
volumes:
- ./postgresql:/docker-entrypoint-initdb.d/
environment:
- POSTGRES_PASSWORD=changeme
mysql:
image: "mysql:8"
ports:
- 3306:3306
volumes:
- ./mysql:/docker-entrypoint-initdb.d/
environment:
- MYSQL_ROOT_PASSWORD=changeme
openldap:
image: docker.io/bitnami/openldap:2.6
ports:
- '389:1389'
- '636:1636'
# docs of these env vars: https://github.com/bitnami/containers/tree/2724f9cd02b3b4e7986a1e2a0b0b30af3737bbd2/bitnami/openldap#configuration
environment:
- LDAP_ROOT=dc=test,dc=com
- LDAP_ADMIN_USERNAME=admin
- LDAP_ADMIN_PASSWORD=password
- LDAP_USERS=user01,user02
- LDAP_PASSWORDS=password1,password2
- LDAP_GROUP=some-ldap-group
volumes:
- 'openldap_data:/bitnami/openldap'

volumes:
openldap_data:
driver: local
31 changes: 31 additions & 0 deletions scripts/mysql/init-db.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
#!/bin/bash

set -euo pipefail

# Number of gradle workers times 4, which was somewhat arbitrary but is sufficient in practice.
# We make extra dbs because a gradle worker ID can exceed the max number of workers.
NUM_OF_DATABASES_TO_CREATE=24


function initDB() {
mysql -uroot -pchangeme <<-EOSQL
SET GLOBAL max_connections = 250;
DROP DATABASE IF EXISTS uaa;
CREATE DATABASE uaa DEFAULT CHARACTER SET utf8mb4;
EOSQL
}

function createDB() {
DATABASE_NAME="uaa_${1}"
echo "Creating MySQL database with name ${DATABASE_NAME}"
mysql -uroot -pchangeme <<-EOSQL
DROP DATABASE IF EXISTS $DATABASE_NAME;
CREATE DATABASE $DATABASE_NAME DEFAULT CHARACTER SET utf8mb4;
EOSQL
}

initDB

for db_id in `seq 1 $NUM_OF_DATABASES_TO_CREATE`; do
createDB $db_id
done
32 changes: 32 additions & 0 deletions scripts/postgresql/init-db.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
#!/usr/bin/env bash

set -euo pipefail

# Number of gradle workers times 4, which was somewhat arbitrary but is sufficient in practice.
# We make extra dbs because a gradle worker ID can exceed the max number of workers.
NUM_OF_DATABASES_TO_CREATE=24

function initDB() {
psql --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
DROP DATABASE IF EXISTS uaa;
CREATE DATABASE uaa;
DROP USER IF EXISTS root;
CREATE USER root WITH SUPERUSER PASSWORD '$POSTGRES_PASSWORD';
SHOW max_connections;
EOSQL
}

function createDB() {
DATABASE_NAME="uaa_${1}"
echo "Creating PostgreSQL database with name ${DATABASE_NAME}"
psql --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
DROP DATABASE IF EXISTS $DATABASE_NAME;
CREATE DATABASE $DATABASE_NAME;
EOSQL
}

initDB

for db_id in `seq 1 $NUM_OF_DATABASES_TO_CREATE`; do
createDB $db_id
done
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
import org.cloudfoundry.identity.uaa.util.beans.PasswordEncoderConfig;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.context.annotation.ImportResource;
import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import org.springframework.test.context.web.WebAppConfiguration;
Expand All @@ -19,7 +18,6 @@
@Retention(RetentionPolicy.RUNTIME)
@ExtendWith(SpringExtension.class)
@ExtendWith(PollutionPreventionExtension.class)
@ActiveProfiles("default")
@WebAppConfiguration
@ContextConfiguration(classes = {
DatabaseOnlyConfiguration.class,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,20 +2,18 @@

import org.cloudfoundry.identity.uaa.test.TestUtils;
import org.flywaydb.core.Flyway;
import org.junit.After;
import org.junit.Before;
import org.junit.runner.RunWith;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.test.context.junit.jupiter.SpringExtension;

import java.sql.SQLException;

import static java.lang.System.getProperties;
import static org.junit.Assume.assumeTrue;

@RunWith(SpringJUnit4ClassRunner.class)
@ExtendWith(SpringExtension.class)
@ContextConfiguration(locations = {
"classpath:spring/env.xml",
"classpath:spring/jdbc-test-base-add-flyway.xml",
Expand All @@ -31,19 +29,14 @@ public abstract class DbMigrationIntegrationTestParent {
MigrationTestRunner migrationTestRunner;
private boolean dbNeedsResetting;

protected abstract String onlyRunTestsForActiveSpringProfileName();

@Before
@BeforeEach
public void setup() {
String active = getProperties().getProperty("spring.profiles.active");
assumeTrue("Expected db profile to be enabled", active != null && active.contains(onlyRunTestsForActiveSpringProfileName()));

dbNeedsResetting = true;
flyway.clean();
migrationTestRunner = new MigrationTestRunner(flyway);
}

@After
@AfterEach
public void cleanup() throws SQLException {
if (dbNeedsResetting) { // cleanup() is always called, even when setup()'s assumeTrue() fails
// Avoid test pollution by putting the db back into a default state that other tests assume
Expand Down
Original file line number Diff line number Diff line change
@@ -1,27 +1,24 @@
package org.cloudfoundry.identity.uaa.db;

import org.junit.Test;
import org.cloudfoundry.identity.uaa.extensions.profiles.DisabledIfProfile;
import org.junit.jupiter.api.Test;

import java.util.List;

import static junit.framework.TestCase.fail;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.hasSize;
import static org.junit.jupiter.api.Assertions.fail;

@DisabledIfProfile({"mysql", "postgresql"})
public class HsqlDbMigrationIntegrationTest extends DbMigrationIntegrationTestParent {

private final String checkPrimaryKeyExists = "SELECT COUNT(*) FROM information_schema.KEY_COLUMN_USAGE WHERE TABLE_SCHEMA = ? AND TABLE_NAME = UPPER(?) AND CONSTRAINT_NAME LIKE 'SYS_PK_%'";
private final String getAllTableNames = "SELECT distinct TABLE_NAME from information_schema.KEY_COLUMN_USAGE where TABLE_SCHEMA = ? and TABLE_NAME != 'schema_version'";
private final String insertNewOauthCodeRecord = "insert into oauth_code(code) values('code');";

@Override
protected String onlyRunTestsForActiveSpringProfileName() {
return "hsqldb";
}

@Test
public void insertMissingPrimaryKeys_onMigrationOnNewDatabase() {
MigrationTest migrationTest = new MigrationTest() {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,29 +1,25 @@
package org.cloudfoundry.identity.uaa.db;

import org.junit.Test;
import org.cloudfoundry.identity.uaa.extensions.profiles.EnabledIfProfile;
import org.junit.jupiter.api.Test;

import java.util.Arrays;
import java.util.List;

import static junit.framework.TestCase.fail;
import static org.hamcrest.CoreMatchers.anyOf;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.hasSize;
import static org.junit.jupiter.api.Assertions.fail;

@EnabledIfProfile("mysql")
public class MySqlDbMigrationIntegrationTest extends DbMigrationIntegrationTestParent {

private final String checkPrimaryKeyExists = "SELECT COUNT(*) FROM information_schema.KEY_COLUMN_USAGE WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? AND CONSTRAINT_NAME = 'PRIMARY'";
private final String getAllTableNames = "SELECT distinct TABLE_NAME from information_schema.KEY_COLUMN_USAGE where TABLE_SCHEMA = ?";
private final String insertNewOauthCodeRecord = "insert into oauth_code(code) values('code');";

@Override
protected String onlyRunTestsForActiveSpringProfileName() {
return "mysql";
}

@Test
public void insertMissingPrimaryKeys_onMigrationOnNewDatabase() {
MigrationTest migrationTest = new MigrationTest() {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,27 +1,23 @@
package org.cloudfoundry.identity.uaa.db;

import org.junit.Test;
import org.cloudfoundry.identity.uaa.extensions.profiles.EnabledIfProfile;
import org.junit.jupiter.api.Test;

import java.util.List;

import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.hasSize;
import static org.junit.Assert.fail;
import static org.junit.jupiter.api.Assertions.fail;

@EnabledIfProfile("postgresql")
public class PostgresDbMigrationIntegrationTest extends DbMigrationIntegrationTestParent {

private final String checkPrimaryKeyExists = "SELECT COUNT(*) FROM information_schema.KEY_COLUMN_USAGE WHERE TABLE_CATALOG = ? AND TABLE_NAME = LOWER(?) AND CONSTRAINT_NAME LIKE LOWER(?)";
private final String getAllTableNames = "SELECT distinct TABLE_NAME from information_schema.KEY_COLUMN_USAGE where TABLE_CATALOG = ? and TABLE_NAME != 'schema_version' AND TABLE_SCHEMA != 'pg_catalog'";
private final String insertNewOauthCodeRecord = "insert into oauth_code(code) values('code');";

@Override
protected String onlyRunTestsForActiveSpringProfileName() {
return "postgresql";
}

@Test
public void everyTableShouldHaveAPrimaryKeyColumn() throws Exception {
flyway.migrate();
Expand Down
Original file line number Diff line number Diff line change
@@ -1,64 +1,76 @@
package org.cloudfoundry.identity.uaa.db;

import org.cloudfoundry.identity.uaa.db.mysql.V1_5_4__NormalizeTableAndColumnNames;
import org.cloudfoundry.identity.uaa.extensions.PollutionPreventionExtension;
import org.cloudfoundry.identity.uaa.extensions.profiles.EnabledIfProfile;
import org.cloudfoundry.identity.uaa.util.beans.PasswordEncoderConfig;
import org.junit.Assume;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContextInitializer;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.context.annotation.ImportResource;
import org.springframework.test.context.ActiveProfiles;
import org.springframework.core.env.MapPropertySource;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import org.springframework.test.context.web.WebAppConfiguration;
import org.springframework.web.context.WebApplicationContext;

import javax.sql.DataSource;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.ResultSet;
import java.util.Arrays;
import java.util.Map;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;

@ImportResource(locations = {
"classpath:spring/env.xml",
"classpath:spring/use_uaa_db_in_mysql_url.xml", // adds this one
"classpath:spring/jdbc-test-base-add-flyway.xml",
"classpath:spring/data-source.xml",
})
class TableAndColumnNormalizationTestConfiguration {
}

/**
* For MySQL, the database name is hardcoded in the {@link V1_5_4__NormalizeTableAndColumnNames} migration as
* {@code uaa}. But the {@link UaaDatabaseName} class dynamically allocates a DB name based on the gradle worker id,
* like {@code uaa_1, uaa_2m ...}.
* <p>
* When the profile is {@code mysql}, hardcode the DB url to have the database name equal to {@code uaa}.
*/
class MySQLInitializer implements ApplicationContextInitializer<ConfigurableApplicationContext> {
@Override
public void initialize(ConfigurableApplicationContext applicationContext) {
var profiles = Arrays.asList(applicationContext.getEnvironment().getActiveProfiles());
if (profiles.contains("mysql")) {
Map<String, Object> dynamicProperties = Map.of("database.url", "jdbc:mysql://127.0.0.1:3306/uaa?useSSL=true&trustServerCertificate=true");
MapPropertySource propertySource = new MapPropertySource("mysql-override", dynamicProperties);
applicationContext.getEnvironment().getPropertySources().addLast(propertySource);
}
}
}

@ExtendWith(SpringExtension.class)
@ExtendWith(PollutionPreventionExtension.class)
@ActiveProfiles("default")
@WebAppConfiguration
@ContextConfiguration(classes = {
TableAndColumnNormalizationTestConfiguration.class,
PasswordEncoderConfig.class,
})
PasswordEncoderConfig.class
},
initializers = MySQLInitializer.class
)
@EnabledIfProfile({"postgresql", "mysql"})
class TableAndColumnNormalizationTest {

private final Logger logger = LoggerFactory.getLogger(getClass());

@Autowired
private DataSource dataSource;

@BeforeEach
void checkMysqlOrPostgresqlProfile(
@Autowired WebApplicationContext webApplicationContext
) {
Assume.assumeTrue(
Arrays.asList(webApplicationContext.getEnvironment().getActiveProfiles()).contains("mysql") ||
Arrays.asList(webApplicationContext.getEnvironment().getActiveProfiles()).contains("postgresql")
);
}

@Test
void checkTables() throws Exception {
try (Connection connection = dataSource.getConnection()) {
Expand Down Expand Up @@ -93,10 +105,11 @@ void checkColumns() throws Exception {
logger.info("Checking column [" + name + "." + col + "]");
if (name != null && DatabaseInformation1_5_3.tableNames.contains(name.toLowerCase())) {
logger.info("Validating column [" + name + "." + col + "]");
assertEquals("Column[%s.%s] is not lower case.".formatted(name, col), col.toLowerCase(), col);
assertEquals(col.toLowerCase(), col, "Column[%s.%s] is not lower case.".formatted(name, col));
}
}
assertTrue(hadSomeResults, "Getting columns from db metadata should have returned some results");
}
}

}
Loading