Skip to content

Commit

Permalink
fix #2863 Add upper limits to search and click log queue sizes
Browse files Browse the repository at this point in the history
  • Loading branch information
@marevol
marevol committed Dec 28, 2024
1 parent 07bd6aa commit e739c8c
Show file tree
Hide file tree
Showing 3 changed files with 76 additions and 12 deletions.
11 changes: 10 additions & 1 deletion src/main/java/org/codelibs/fess/helper/SearchLogHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,12 +130,16 @@ public UserInfo load(final String key) throws Exception {
*/
public void addSearchLog(final SearchRequestParams params, final LocalDateTime requestedTime, final String queryId, final String query,
final int pageStart, final int pageSize, final QueryResponseList queryResponseList) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
if (searchLogQueue.size() > fessConfig.getLoggingSearchMaxQueueSizeAsInteger()) {
logger.warn("[{}] The search log queue size is too large. Skipped the search log: {}", queryId, query);
return;
}

final RoleQueryHelper roleQueryHelper = ComponentUtil.getRoleQueryHelper();
final UserInfoHelper userInfoHelper = ComponentUtil.getUserInfoHelper();
final SearchLog searchLog = new SearchLog();

final FessConfig fessConfig = ComponentUtil.getFessConfig();
if (fessConfig.isUserInfo()) {
final String userCode = userInfoHelper.getUserCode();
if (userCode != null) {
Expand Down Expand Up @@ -235,6 +239,11 @@ protected void addDocumentsInResponse(final QueryResponseList queryResponseList,
* @param clickLog The click log.
*/
public void addClickLog(final ClickLog clickLog) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
if (clickLogQueue.size() > fessConfig.getLoggingClickMaxQueueSizeAsInteger()) {
logger.warn("[{}] The click log queue size is too large. Skipped the click log: {} {}", clickLog);
return;
}
clickLogQueue.add(clickLog);
}

Expand Down
73 changes: 63 additions & 10 deletions src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1200,6 +1200,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/** The key of the configuration. e.g. 60000 */
String INDEX_BACKUP_LOG_LOAD_TIMEOUT = "index.backup.log.load.timeout";

/** The key of the configuration. e.g. org.codelibs,org.dbflute,org.lastaflute */
String LOGGING_APP_PACKAGES = "logging.app.packages";

/** The key of the configuration. e.g. true */
String LOGGING_SEARCH_DOCS_ENABLED = "logging.search.docs.enabled";

Expand All @@ -1209,8 +1212,11 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/** The key of the configuration. e.g. true */
String LOGGING_SEARCH_USE_LOGFILE = "logging.search.use.logfile";

/** The key of the configuration. e.g. org.codelibs,org.dbflute,org.lastaflute */
String LOGGING_APP_PACKAGES = "logging.app.packages";
/** The key of the configuration. e.g. 10000 */
String LOGGING_SEARCH_MAX_QUEUE_SIZE = "logging.search.max.queue.size";

/** The key of the configuration. e.g. 10000 */
String LOGGING_CLICK_MAX_QUEUE_SIZE = "logging.click.max.queue.size";

/** The key of the configuration. e.g. 10000 */
String FORM_ADMIN_MAX_INPUT_SIZE = "form.admin.max.input.size";
Expand Down Expand Up @@ -5636,18 +5642,24 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
*/
Integer getIndexBackupLogLoadTimeoutAsInteger();

/**
* Get the value for the key 'logging.app.packages'. <br>
* The value is, e.g. org.codelibs,org.dbflute,org.lastaflute <br>
* comment: logging
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getLoggingAppPackages();

/**
* Get the value for the key 'logging.search.docs.enabled'. <br>
* The value is, e.g. true <br>
* comment: logging
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getLoggingSearchDocsEnabled();

/**
* Is the property for the key 'logging.search.docs.enabled' true? <br>
* The value is, e.g. true <br>
* comment: logging
* @return The determination, true or false. (if not found, exception but basically no way)
*/
boolean isLoggingSearchDocsEnabled();
Expand All @@ -5674,11 +5686,34 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
boolean isLoggingSearchUseLogfile();

/**
* Get the value for the key 'logging.app.packages'. <br>
* The value is, e.g. org.codelibs,org.dbflute,org.lastaflute <br>
* Get the value for the key 'logging.search.max.queue.size'. <br>
* The value is, e.g. 10000 <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getLoggingAppPackages();
String getLoggingSearchMaxQueueSize();

/**
* Get the value for the key 'logging.search.max.queue.size' as {@link Integer}. <br>
* The value is, e.g. 10000 <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
* @throws NumberFormatException When the property is not integer.
*/
Integer getLoggingSearchMaxQueueSizeAsInteger();

/**
* Get the value for the key 'logging.click.max.queue.size'. <br>
* The value is, e.g. 10000 <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getLoggingClickMaxQueueSize();

/**
* Get the value for the key 'logging.click.max.queue.size' as {@link Integer}. <br>
* The value is, e.g. 10000 <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
* @throws NumberFormatException When the property is not integer.
*/
Integer getLoggingClickMaxQueueSizeAsInteger();

/**
* Get the value for the key 'form.admin.max.input.size'. <br>
Expand Down Expand Up @@ -9813,6 +9848,10 @@ public Integer getIndexBackupLogLoadTimeoutAsInteger() {
return getAsInteger(FessConfig.INDEX_BACKUP_LOG_LOAD_TIMEOUT);
}

public String getLoggingAppPackages() {
return get(FessConfig.LOGGING_APP_PACKAGES);
}

public String getLoggingSearchDocsEnabled() {
return get(FessConfig.LOGGING_SEARCH_DOCS_ENABLED);
}
Expand All @@ -9833,8 +9872,20 @@ public boolean isLoggingSearchUseLogfile() {
return is(FessConfig.LOGGING_SEARCH_USE_LOGFILE);
}

public String getLoggingAppPackages() {
return get(FessConfig.LOGGING_APP_PACKAGES);
public String getLoggingSearchMaxQueueSize() {
return get(FessConfig.LOGGING_SEARCH_MAX_QUEUE_SIZE);
}

public Integer getLoggingSearchMaxQueueSizeAsInteger() {
return getAsInteger(FessConfig.LOGGING_SEARCH_MAX_QUEUE_SIZE);
}

public String getLoggingClickMaxQueueSize() {
return get(FessConfig.LOGGING_CLICK_MAX_QUEUE_SIZE);
}

public Integer getLoggingClickMaxQueueSizeAsInteger() {
return getAsInteger(FessConfig.LOGGING_CLICK_MAX_QUEUE_SIZE);
}

public String getFormAdminMaxInputSize() {
Expand Down Expand Up @@ -11373,11 +11424,13 @@ protected java.util.Map<String, String> prepareGeneratedDefaultMap() {
"fess_basic_config.bulk,fess_config.bulk,fess_user.bulk,system.properties,fess.json,doc.json");
defaultMap.put(FessConfig.INDEX_BACKUP_LOG_TARGETS, "click_log.ndjson,favorite_log.ndjson,search_log.ndjson,user_info.ndjson");
defaultMap.put(FessConfig.INDEX_BACKUP_LOG_LOAD_TIMEOUT, "60000");
defaultMap.put(FessConfig.LOGGING_APP_PACKAGES, "org.codelibs,org.dbflute,org.lastaflute");
defaultMap.put(FessConfig.LOGGING_SEARCH_DOCS_ENABLED, "true");
defaultMap.put(FessConfig.LOGGING_SEARCH_DOCS_FIELDS,
"filetype,created,click_count,title,doc_id,url,score,site,filename,host,digest,boost,mimetype,favorite_count,_id,lang,last_modified,content_length,timestamp");
defaultMap.put(FessConfig.LOGGING_SEARCH_USE_LOGFILE, "true");
defaultMap.put(FessConfig.LOGGING_APP_PACKAGES, "org.codelibs,org.dbflute,org.lastaflute");
defaultMap.put(FessConfig.LOGGING_SEARCH_MAX_QUEUE_SIZE, "10000");
defaultMap.put(FessConfig.LOGGING_CLICK_MAX_QUEUE_SIZE, "10000");
defaultMap.put(FessConfig.FORM_ADMIN_MAX_INPUT_SIZE, "10000");
defaultMap.put(FessConfig.FORM_ADMIN_LABEL_IN_CONFIG_ENABLED, "false");
defaultMap.put(FessConfig.FORM_ADMIN_DEFAULT_TEMPLATE_NAME, "__TEMPLATE__");
Expand Down
4 changes: 3 additions & 1 deletion src/main/resources/fess_config.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -632,10 +632,12 @@ index.backup.log.targets=click_log.ndjson,favorite_log.ndjson,search_log.ndjson,
index.backup.log.load.timeout=60000

# logging
logging.app.packages=org.codelibs,org.dbflute,org.lastaflute
logging.search.docs.enabled=true
logging.search.docs.fields=filetype,created,click_count,title,doc_id,url,score,site,filename,host,digest,boost,mimetype,favorite_count,_id,lang,last_modified,content_length,timestamp
logging.search.use.logfile=true
logging.app.packages=org.codelibs,org.dbflute,org.lastaflute
logging.search.max.queue.size=10000
logging.click.max.queue.size=10000

# ========================================================================================
# Web
Expand Down

0 comments on commit e739c8c

Please sign in to comment.