diff --git a/addOns/client/client.gradle.kts b/addOns/client/client.gradle.kts index 45847bd8f78..498bc5fd64c 100644 --- a/addOns/client/client.gradle.kts +++ b/addOns/client/client.gradle.kts @@ -19,6 +19,18 @@ zapAddOn { } } } + register("org.zaproxy.addon.client.automation.ExtensionClientAutomation") { + classnames { + allowed.set(listOf("org.zaproxy.addon.client.automation")) + } + dependencies { + addOns { + register("automation") { + version.set(">=0.43.0") + } + } + } + } } dependencies { addOns { @@ -45,6 +57,7 @@ crowdin { } dependencies { + zapAddOn("automation") zapAddOn("commonlib") zapAddOn("selenium") zapAddOn("network") diff --git a/addOns/client/src/main/java/org/zaproxy/addon/client/ClientOptions.java b/addOns/client/src/main/java/org/zaproxy/addon/client/ClientOptions.java index 0c13dd5a524..e03bf57cd90 100644 --- a/addOns/client/src/main/java/org/zaproxy/addon/client/ClientOptions.java +++ b/addOns/client/src/main/java/org/zaproxy/addon/client/ClientOptions.java @@ -23,6 +23,7 @@ import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.parosproxy.paros.Constant; import org.zaproxy.addon.commonlib.Constants; import org.zaproxy.zap.common.VersionedAbstractParam; import org.zaproxy.zap.extension.api.ZapApiIgnore; @@ -36,6 +37,12 @@ public class ClientOptions extends VersionedAbstractParam { static final String CLIENT_BASE_KEY = "client"; + public static final String DEFAULT_BROWSER_ID = Browser.FIREFOX_HEADLESS.getId(); + public static final int DEFAULT_MAX_DEPTH = 5; + public static final int DEFAULT_INITIAL_LOAD_TIME = 5; + public static final int DEFAULT_PAGE_LOAD_TIME = 1; + public static final int DEFAULT_SHUTDOWN_TIME = 5; + private static final String CONFIG_VERSION_KEY = CLIENT_BASE_KEY + VERSION_ATTRIBUTE; private static final String PSCAN_ENABLED_KEY = CLIENT_BASE_KEY + ".pscanEnabled"; private static final String PSCAN_DISABLED_RULES_KEY = CLIENT_BASE_KEY + ".pscanRulesDisabled"; @@ -50,18 +57,16 @@ public class ClientOptions extends VersionedAbstractParam { private static final String MAX_CHILDREN_KEY = CLIENT_BASE_KEY + ".maxChildren"; private static final String MAX_SCANS_IN_UI_KEY = CLIENT_BASE_KEY + ".maxScansInUI"; - private static final String DEFAULT_BROWSER_ID = Browser.FIREFOX_HEADLESS.getId(); - private String browserId; private int threadCount; - private int initialLoadTimeInSecs; - private int pageLoadTimeInSecs; - private int shutdownTimeInSecs; + private int initialLoadTimeInSecs = DEFAULT_INITIAL_LOAD_TIME; + private int pageLoadTimeInSecs = DEFAULT_PAGE_LOAD_TIME; + private int shutdownTimeInSecs = DEFAULT_SHUTDOWN_TIME; private boolean pscanEnabled; private List pscanRulesDisabled; private boolean showAdvancedDialog; private int maxChildren; - private int maxDepth = 5; + private int maxDepth = DEFAULT_MAX_DEPTH; private int maxDuration; private int maxScansInUi = 5; @@ -75,11 +80,11 @@ protected void parseImpl() { this.pscanEnabled = getBoolean(PSCAN_ENABLED_KEY, true); this.browserId = getString(BROWSER_ID_KEY, DEFAULT_BROWSER_ID); this.threadCount = Math.max(1, getInt(THREAD_COUNT_KEY, Constants.getDefaultThreadCount())); - this.initialLoadTimeInSecs = getInt(INITIAL_LOAD_TIME_KEY, 5); - this.pageLoadTimeInSecs = getInt(PAGE_LOAD_TIME_KEY, 1); - this.shutdownTimeInSecs = getInt(SHUTDOWN_TIME_KEY, 5); + this.initialLoadTimeInSecs = getInt(INITIAL_LOAD_TIME_KEY, DEFAULT_INITIAL_LOAD_TIME); + this.pageLoadTimeInSecs = getInt(PAGE_LOAD_TIME_KEY, DEFAULT_PAGE_LOAD_TIME); + this.shutdownTimeInSecs = getInt(SHUTDOWN_TIME_KEY, DEFAULT_SHUTDOWN_TIME); this.maxChildren = getInt(MAX_CHILDREN_KEY, 0); - this.maxDepth = getInt(MAX_DEPTH_KEY, 5); + this.maxDepth = getInt(MAX_DEPTH_KEY, DEFAULT_MAX_DEPTH); this.maxDuration = getInt(MAX_DURATION_KEY, 0); this.maxScansInUi = getInt(MAX_SCANS_IN_UI_KEY, 5); @@ -162,6 +167,9 @@ public int getThreadCount() { } public void setThreadCount(int threadCount) { + if (threadCount <= 0) { + threadCount = Constant.getDefaultThreadCount(); + } this.threadCount = threadCount; getConfig().setProperty(THREAD_COUNT_KEY, threadCount); } diff --git a/addOns/client/src/main/java/org/zaproxy/addon/client/automation/ClientSpiderJob.java b/addOns/client/src/main/java/org/zaproxy/addon/client/automation/ClientSpiderJob.java new file mode 100644 index 00000000000..d7281f822fc --- /dev/null +++ b/addOns/client/src/main/java/org/zaproxy/addon/client/automation/ClientSpiderJob.java @@ -0,0 +1,290 @@ +/* + * Zed Attack Proxy (ZAP) and its related class files. + * + * ZAP is an HTTP/HTTPS proxy for assessing web application security. + * + * Copyright 2024 The ZAP Development Team + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.zaproxy.addon.client.automation; + +import java.util.Map; +import java.util.concurrent.TimeUnit; +import lombok.Getter; +import lombok.Setter; +import org.apache.commons.configuration.XMLConfiguration; +import org.apache.commons.httpclient.URIException; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.parosproxy.paros.Constant; +import org.parosproxy.paros.control.Control; +import org.zaproxy.addon.automation.AutomationData; +import org.zaproxy.addon.automation.AutomationEnvironment; +import org.zaproxy.addon.automation.AutomationJob; +import org.zaproxy.addon.automation.AutomationProgress; +import org.zaproxy.addon.automation.ContextWrapper; +import org.zaproxy.addon.automation.jobs.JobData; +import org.zaproxy.addon.automation.jobs.JobUtils; +import org.zaproxy.addon.client.ClientOptions; +import org.zaproxy.addon.client.ExtensionClientIntegration; +import org.zaproxy.addon.client.spider.ClientSpider; +import org.zaproxy.addon.commonlib.Constants; +import org.zaproxy.zap.users.User; + +public class ClientSpiderJob extends AutomationJob { + + private static final Logger LOGGER = LogManager.getLogger(ClientSpiderJob.class); + + private static final String JOB_NAME = "spiderClient"; + + private ExtensionClientIntegration extSpider; + + private Data data; + private Parameters parameters = new Parameters(); + + public ClientSpiderJob() { + this.data = new Data(this, parameters); + } + + private ExtensionClientIntegration getExtClient() { + if (extSpider == null) { + extSpider = + Control.getSingleton() + .getExtensionLoader() + .getExtension(ExtensionClientIntegration.class); + } + return extSpider; + } + + @Override + public void verifyParameters(AutomationProgress progress) { + Map jobData = this.getJobData(); + if (jobData == null) { + return; + } + Map parametersData = (Map) jobData.get("parameters"); + JobUtils.applyParamsToObject( + parametersData, this.parameters, this.getName(), new String[] {}, progress); + } + + @Override + public void applyParameters(AutomationProgress progress) { + // Nothing to do + } + + @Override + public boolean supportsMonitorTests() { + return true; + } + + @Override + public void runJob(AutomationEnvironment env, AutomationProgress progress) { + + ContextWrapper context = getContextWrapper(env, progress); + if (context == null) { + return; + } + + User user = this.getUser(this.getParameters().getUser(), progress); + + String uriStr = this.getParameters().getUrl(); + if (StringUtils.isEmpty(uriStr)) { + uriStr = context.getUrls().get(0); + } + uriStr = env.replaceVars(uriStr); + + int scanId = -1; + try { + scanId = + getExtClient() + .startScan( + uriStr, paramsToOptions(), context.getContext(), user, false); + } catch (URIException e) { + progress.error(Constant.messages.getString("automation.error.context.badurl", uriStr)); + return; + } catch (Exception e) { + progress.error( + Constant.messages.getString( + "automation.error.unexpected.internal", e.getMessage())); + LOGGER.error(e.getMessage(), e); + return; + } + ClientSpider spider = getExtClient().getScan(scanId); + + long endTime = Long.MAX_VALUE; + if (JobUtils.unBox(this.getParameters().getMaxDuration()) > 0) { + // The spider should stop, if it doesnt we will stop it (after a few seconds leeway) + endTime = + System.currentTimeMillis() + + TimeUnit.MINUTES.toMillis(this.getParameters().getMaxDuration()) + + TimeUnit.SECONDS.toMillis(5); + } + + // Wait for the client spider to finish + boolean forceStop = false; + + while (true) { + this.sleep(500); + + if (!spider.isRunning()) { + break; + } + if (!this.runMonitorTests(progress) || System.currentTimeMillis() > endTime) { + forceStop = true; + break; + } + } + if (forceStop) { + spider.stopScan(); + progress.info(Constant.messages.getString("automation.info.jobstopped", getType())); + } + } + + protected ClientOptions paramsToOptions() { + ClientOptions options = new ClientOptions(); + options.load(new XMLConfiguration()); + + if (!StringUtils.isBlank(this.parameters.getBrowserId())) { + options.setBrowserId(this.parameters.getBrowserId()); + } + if (this.parameters.getMaxDuration() != null) { + options.setMaxDuration(this.parameters.getMaxDuration()); + } + if (this.parameters.getMaxChildren() != null) { + options.setMaxChildren(this.parameters.getMaxChildren()); + } + if (this.parameters.getMaxCrawlDepth() != null) { + options.setMaxDepth(this.parameters.getMaxCrawlDepth()); + } + if (this.parameters.getNumberOfBrowsers() != null) { + options.setThreadCount(this.parameters.getNumberOfBrowsers()); + } + if (this.parameters.getInitialLoadTime() != null) { + options.setInitialLoadTimeInSecs(this.parameters.getInitialLoadTime()); + } + if (this.parameters.getPageLoadTime() != null) { + options.setPageLoadTimeInSecs(this.parameters.getPageLoadTime()); + } + if (this.parameters.getShutdownTime() != null) { + options.setShutdownTimeInSecs(this.parameters.getShutdownTime()); + } + return options; + } + + private ContextWrapper getContextWrapper( + AutomationEnvironment env, AutomationProgress progress) { + String contextName = this.getParameters().getContext(); + if (StringUtils.isEmpty(contextName)) { + return env.getDefaultContextWrapper(); + } + + ContextWrapper wrapper = env.getContextWrapper(contextName); + if (wrapper != null) { + return wrapper; + } + + progress.error( + Constant.messages.getString("automation.error.context.unknown", contextName)); + return null; + } + + @Override + public String getTemplateDataMin() { + return ExtensionClientAutomation.getResourceAsString(this.getType() + "-min.yaml"); + } + + @Override + public String getTemplateDataMax() { + return ExtensionClientAutomation.getResourceAsString(this.getType() + "-max.yaml"); + } + + @Override + public Order getOrder() { + return Order.LAST_EXPLORE; + } + + @Override + public Object getParamMethodObject() { + return null; + } + + @Override + public String getParamMethodName() { + return null; + } + + @Override + public String getType() { + return JOB_NAME; + } + + @Override + public void showDialog() { + new ClientSpiderJobDialog(this).setVisible(true); + } + + @Override + public String getSummary() { + String context = this.getParameters().getContext(); + if (StringUtils.isEmpty(context)) { + context = Constant.messages.getString("client.automation.default"); + } + return Constant.messages.getString( + "client.automation.dialog.summary", + context, + JobUtils.unBox(this.getParameters().getUrl(), "''")); + } + + @Override + public Data getData() { + return data; + } + + @Override + public Parameters getParameters() { + return parameters; + } + + public static class Data extends JobData { + private Parameters parameters; + + public Data(AutomationJob job, Parameters parameters) { + super(job); + this.parameters = parameters; + } + + public Parameters getParameters() { + return parameters; + } + } + + @Getter + @Setter + public static class Parameters extends AutomationData { + private String context = ""; + private String user = ""; + private String url = ""; + private Integer maxDuration; + private Integer maxChildren; + private Integer maxCrawlDepth = ClientOptions.DEFAULT_MAX_DEPTH; + private Integer numberOfBrowsers = Constants.getDefaultThreadCount(); + private String browserId; + private Integer initialLoadTime = ClientOptions.DEFAULT_INITIAL_LOAD_TIME; + private Integer pageLoadTime = ClientOptions.DEFAULT_PAGE_LOAD_TIME; + private Integer shutdownTime = ClientOptions.DEFAULT_SHUTDOWN_TIME; + + public Parameters() {} + } +} diff --git a/addOns/client/src/main/java/org/zaproxy/addon/client/automation/ClientSpiderJobDialog.java b/addOns/client/src/main/java/org/zaproxy/addon/client/automation/ClientSpiderJobDialog.java new file mode 100644 index 00000000000..e98a2352431 --- /dev/null +++ b/addOns/client/src/main/java/org/zaproxy/addon/client/automation/ClientSpiderJobDialog.java @@ -0,0 +1,247 @@ +/* + * Zed Attack Proxy (ZAP) and its related class files. + * + * ZAP is an HTTP/HTTPS proxy for assessing web application security. + * + * Copyright 2024 The ZAP Development Team + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.zaproxy.addon.client.automation; + +import java.awt.Component; +import java.util.ArrayList; +import java.util.List; +import javax.swing.JTextField; +import org.parosproxy.paros.control.Control; +import org.parosproxy.paros.view.View; +import org.zaproxy.addon.client.ClientOptions; +import org.zaproxy.addon.client.automation.ClientSpiderJob.Parameters; +import org.zaproxy.addon.commonlib.Constants; +import org.zaproxy.zap.extension.selenium.ExtensionSelenium; +import org.zaproxy.zap.extension.selenium.ProvidedBrowserUI; +import org.zaproxy.zap.utils.DisplayUtils; +import org.zaproxy.zap.view.StandardFieldsDialog; + +@SuppressWarnings("serial") +public class ClientSpiderJobDialog extends StandardFieldsDialog { + + private static final long serialVersionUID = 1L; + + private static final String[] TAB_LABELS = { + "client.automation.dialog.tab.params", "client.automation.dialog.spider.tab.adv" + }; + + private static final String TITLE = "client.automation.dialog.spider.title"; + private static final String NAME_PARAM = "client.automation.dialog.spider.name"; + private static final String CONTEXT_PARAM = "client.automation.dialog.spider.context"; + private static final String USER_PARAM = "client.automation.dialog.spider.user"; + private static final String URL_PARAM = "client.automation.dialog.spider.url"; + private static final String MAX_DURATION_PARAM = "client.automation.dialog.spider.maxduration"; + private static final String MAX_CRAWL_DEPTH_PARAM = + "client.automation.dialog.spider.maxcrawldepth"; + private static final String NUM_BROWSERS_PARAM = "client.automation.dialog.spider.numbrowsers"; + private static final String BROWSER_ID_PARAM = "client.automation.dialog.spider.browserid"; + private static final String FIELD_ADVANCED = "client.automation.dialog.spider.advanced"; + + private static final String MAX_CHILDREN_PARAM = "client.automation.dialog.spider.maxchildren"; + private static final String INITIAL_PAGE_LOADTIME_PARAM = + "client.automation.dialog.spider.initialtime"; + private static final String PAGE_LOADTIME_PARAM = "client.automation.dialog.spider.loadtime"; + private static final String SHUTDOWN_TIME_PARAM = + "client.automation.dialog.spider.shutdowntime"; + + private ClientSpiderJob job; + private ExtensionSelenium extSel = null; + + public ClientSpiderJobDialog(ClientSpiderJob job) { + super( + View.getSingleton().getMainFrame(), + TITLE, + DisplayUtils.getScaledDimension(450, 350), + TAB_LABELS); + this.job = job; + this.addTextField(0, NAME_PARAM, this.job.getName()); + List contextNames = this.job.getEnv().getContextNames(); + // Add blank option + contextNames.add(0, ""); + this.addComboField(0, CONTEXT_PARAM, contextNames, this.job.getParameters().getContext()); + + List users = job.getEnv().getAllUserNames(); + // Add blank option + users.add(0, ""); + this.addComboField(0, USER_PARAM, users, this.job.getParameters().getUser()); + + // Cannot select the node as it might not be present in the Sites tree + this.addNodeSelectField(0, URL_PARAM, null, true, false); + Component urlField = this.getField(URL_PARAM); + if (urlField instanceof JTextField) { + ((JTextField) urlField).setText(this.job.getParameters().getUrl()); + } + + List browserList = getExtSelenium().getProvidedBrowserUIList(); + List browserNames = new ArrayList<>(); + String defaultBrowser = ""; + browserNames.add(""); // Default to empty + for (ProvidedBrowserUI browser : browserList) { + browserNames.add(browser.getName()); + if (browser.getBrowser().getId().equals(this.job.getParameters().getBrowserId())) { + defaultBrowser = browser.getName(); + } + } + this.addComboField(0, BROWSER_ID_PARAM, browserNames, defaultBrowser); + + this.addCheckBoxField(0, FIELD_ADVANCED, advOptionsSet()); + this.addFieldListener(FIELD_ADVANCED, e -> setAdvancedTabs(getBoolValue(FIELD_ADVANCED))); + + this.addPadding(0); + + this.addNumberField( + 1, + NUM_BROWSERS_PARAM, + 1, + Integer.MAX_VALUE, + getInt( + this.job.getParameters().getNumberOfBrowsers(), + Constants.getDefaultThreadCount())); + + this.addNumberField( + 1, + MAX_CRAWL_DEPTH_PARAM, + 0, + Integer.MAX_VALUE, + getInt( + this.job.getParameters().getMaxCrawlDepth(), + ClientOptions.DEFAULT_MAX_DEPTH)); + + this.addNumberField( + 1, + MAX_CHILDREN_PARAM, + 0, + Integer.MAX_VALUE, + getInt(this.job.getParameters().getMaxChildren(), 0)); + this.addNumberField( + 1, + INITIAL_PAGE_LOADTIME_PARAM, + 0, + Integer.MAX_VALUE, + getInt( + this.job.getParameters().getInitialLoadTime(), + ClientOptions.DEFAULT_INITIAL_LOAD_TIME)); + this.addNumberField( + 1, + PAGE_LOADTIME_PARAM, + 0, + Integer.MAX_VALUE, + getInt( + this.job.getParameters().getPageLoadTime(), + ClientOptions.DEFAULT_PAGE_LOAD_TIME)); + this.addNumberField( + 1, + SHUTDOWN_TIME_PARAM, + 0, + Integer.MAX_VALUE, + getInt( + this.job.getParameters().getShutdownTime(), + ClientOptions.DEFAULT_SHUTDOWN_TIME)); + this.addNumberField( + 1, + MAX_DURATION_PARAM, + 1, + Integer.MAX_VALUE, + getInt(this.job.getParameters().getMaxDuration(), 0)); + + this.addPadding(1); + + setAdvancedTabs(getBoolValue(FIELD_ADVANCED)); + } + + private int getInt(Integer i, int defaultValue) { + if (i == null) { + return defaultValue; + } + return i.intValue(); + } + + private boolean advOptionsSet() { + Parameters params = this.job.getParameters(); + return params.getBrowserId() != null + || params.getMaxCrawlDepth() != null + || params.getMaxChildren() != null + || params.getInitialLoadTime() != null + || params.getPageLoadTime() != null + || params.getShutdownTime() != null + || params.getMaxDuration() != null; + } + + private void setAdvancedTabs(boolean visible) { + // Show/hide all except from the first tab + this.setTabsVisible(new String[] {"client.automation.dialog.spider.tab.adv"}, visible); + } + + private ExtensionSelenium getExtSelenium() { + if (extSel == null) { + extSel = + Control.getSingleton() + .getExtensionLoader() + .getExtension(ExtensionSelenium.class); + } + return extSel; + } + + @Override + public void save() { + this.job.setName(this.getStringValue(NAME_PARAM)); + this.job.getParameters().setContext(this.getStringValue(CONTEXT_PARAM)); + this.job.getParameters().setUser(this.getStringValue(USER_PARAM)); + this.job.getParameters().setUrl(this.getStringValue(URL_PARAM)); + String browserName = this.getStringValue(BROWSER_ID_PARAM); + if (browserName.isEmpty()) { + this.job.getParameters().setBrowserId(null); + } else { + List browserList = getExtSelenium().getProvidedBrowserUIList(); + for (ProvidedBrowserUI bui : browserList) { + if (browserName.equals(bui.getName())) { + this.job.getParameters().setBrowserId(bui.getBrowser().getId()); + break; + } + } + } + + if (this.getBoolValue(FIELD_ADVANCED)) { + this.job.getParameters().setNumberOfBrowsers(this.getIntValue(NUM_BROWSERS_PARAM)); + this.job.getParameters().setMaxCrawlDepth(this.getIntValue(MAX_CRAWL_DEPTH_PARAM)); + this.job.getParameters().setMaxChildren(this.getIntValue(MAX_CHILDREN_PARAM)); + this.job + .getParameters() + .setInitialLoadTime(this.getIntValue(INITIAL_PAGE_LOADTIME_PARAM)); + this.job.getParameters().setPageLoadTime(this.getIntValue(PAGE_LOADTIME_PARAM)); + this.job.getParameters().setShutdownTime(this.getIntValue(SHUTDOWN_TIME_PARAM)); + this.job.getParameters().setMaxDuration(this.getIntValue(MAX_DURATION_PARAM)); + } else { + this.job.getParameters().setNumberOfBrowsers(null); + this.job.getParameters().setMaxCrawlDepth(null); + this.job.getParameters().setMaxChildren(null); + this.job.getParameters().setInitialLoadTime(null); + this.job.getParameters().setPageLoadTime(null); + this.job.getParameters().setShutdownTime(null); + this.job.getParameters().setMaxDuration(null); + } + this.job.setChanged(); + } + + @Override + public String validateFields() { + return null; + } +} diff --git a/addOns/client/src/main/java/org/zaproxy/addon/client/automation/ExtensionClientAutomation.java b/addOns/client/src/main/java/org/zaproxy/addon/client/automation/ExtensionClientAutomation.java new file mode 100644 index 00000000000..d420dfaed5a --- /dev/null +++ b/addOns/client/src/main/java/org/zaproxy/addon/client/automation/ExtensionClientAutomation.java @@ -0,0 +1,107 @@ +/* + * Zed Attack Proxy (ZAP) and its related class files. + * + * ZAP is an HTTP/HTTPS proxy for assessing web application security. + * + * Copyright 2024 The ZAP Development Team + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.zaproxy.addon.client.automation; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.List; +import java.util.stream.Collectors; +import org.parosproxy.paros.CommandLine; +import org.parosproxy.paros.Constant; +import org.parosproxy.paros.control.Control; +import org.parosproxy.paros.extension.Extension; +import org.parosproxy.paros.extension.ExtensionAdaptor; +import org.parosproxy.paros.extension.ExtensionHook; +import org.zaproxy.addon.automation.ExtensionAutomation; +import org.zaproxy.addon.client.ExtensionClientIntegration; + +public class ExtensionClientAutomation extends ExtensionAdaptor { + + public static final String NAME = "ExtensionClientAutomation"; + + private static final String RESOURCES_DIR = "/org/zaproxy/addon/client/resources/"; + + private static final List> DEPENDENCIES = + List.of(ExtensionClientIntegration.class, ExtensionAutomation.class); + + private ClientSpiderJob job; + + public ExtensionClientAutomation() { + super(NAME); + } + + @Override + public boolean supportsDb(String type) { + return true; + } + + @Override + public void hook(ExtensionHook extensionHook) { + super.hook(extensionHook); + ExtensionAutomation extAuto = + Control.getSingleton().getExtensionLoader().getExtension(ExtensionAutomation.class); + job = new ClientSpiderJob(); + extAuto.registerAutomationJob(job); + } + + @Override + public boolean canUnload() { + return true; + } + + @Override + public void unload() { + ExtensionAutomation extAuto = + Control.getSingleton().getExtensionLoader().getExtension(ExtensionAutomation.class); + + extAuto.unregisterAutomationJob(job); + } + + @Override + public List> getDependencies() { + return DEPENDENCIES; + } + + public static String getResourceAsString(String name) { + try (InputStream in = + ExtensionClientIntegration.class.getResourceAsStream(RESOURCES_DIR + name)) { + return new BufferedReader(new InputStreamReader(in)) + .lines() + .collect(Collectors.joining("\n")) + + "\n"; + } catch (Exception e) { + CommandLine.error( + Constant.messages.getString( + "client.automation.error.nofile", RESOURCES_DIR + name)); + } + return ""; + } + + @Override + public String getDescription() { + return Constant.messages.getString("client.automation.desc"); + } + + @Override + public String getUIName() { + return Constant.messages.getString("client.automation.name"); + } +} diff --git a/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpider.java b/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpider.java index 94b64bcc3fc..e1af505a2ac 100644 --- a/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpider.java +++ b/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpider.java @@ -92,7 +92,6 @@ public class ClientSpider implements EventConsumer, GenericScanner2 { * * The following features should be implemented in future releases: * Clicking on likely navigation elements - * Automation framework support * API support */ private static final Logger LOGGER = LogManager.getLogger(ClientSpider.class); diff --git a/addOns/client/src/main/resources/org/zaproxy/addon/client/resources/Messages.properties b/addOns/client/src/main/resources/org/zaproxy/addon/client/resources/Messages.properties index e779292d995..992846a27f6 100644 --- a/addOns/client/src/main/resources/org/zaproxy/addon/client/resources/Messages.properties +++ b/addOns/client/src/main/resources/org/zaproxy/addon/client/resources/Messages.properties @@ -1,6 +1,28 @@ client.activeActionPrefix = Client Spidering: {0} client.attack.spider = Client Spider... +client.automation.default = Default + +client.automation.desc = Client Spider Automation Framework Integration +client.automation.dialog.spider.advanced = Show Advanced Options: +client.automation.dialog.spider.browserid = Browser ID: +client.automation.dialog.spider.context = Context: +client.automation.dialog.spider.initialtime = Initial Page Load Time: +client.automation.dialog.spider.loadtime = Page Load Time: +client.automation.dialog.spider.maxchildren = Maximum Children: +client.automation.dialog.spider.maxcrawldepth = Max Crawl Depth: +client.automation.dialog.spider.maxduration = Max Duration: +client.automation.dialog.spider.name = Job Name: +client.automation.dialog.spider.numbrowsers = Number of Browsers: +client.automation.dialog.spider.shutdowntime = Shutdown Time: +client.automation.dialog.spider.tab.adv = Options +client.automation.dialog.spider.title = Client Spider +client.automation.dialog.spider.url = URL: +client.automation.dialog.spider.user = User: +client.automation.dialog.summary = Default + +client.automation.dialog.tab.params = Scope +client.automation.name = Client Spider Automation client.components.table.header.form = Form ID client.components.table.header.href = HREF diff --git a/addOns/client/src/main/resources/org/zaproxy/addon/client/resources/spiderClient-max.yaml b/addOns/client/src/main/resources/org/zaproxy/addon/client/resources/spiderClient-max.yaml new file mode 100644 index 00000000000..55152973c9f --- /dev/null +++ b/addOns/client/src/main/resources/org/zaproxy/addon/client/resources/spiderClient-max.yaml @@ -0,0 +1,13 @@ + - type: spiderClient # The client spider - a spider which explores modern web apps more effectively + parameters: + context: # String: Name of the context to spider, default: first context + user: # String: An optional user to use for authentication, must be defined in the env + url: # String: URL to start spidering from, default: first context URL + maxDuration: # Int: The max time in minutes the spider will be allowed to run for, default: 0 unlimited + maxCrawlDepth: # Int: The maximum tree depth to explore, default 5 + maxChildren: # Int: The maximum number of children to add to each node in the tree + numberOfBrowsers: # Int: The number of browsers the spider will use, more will be faster but will use up more memory, default 2 x number of cores + browserId: # String: Browser ID to use, default: firefox-headless + initialLoadTime: # Int: The time in seconds to wait after the initial URL is loaded, default: 5 + pageLoadTime: # Int: The time in seconds to wait after a new URL is loaded, default: 1 + shutdownTime: # Int: The time in seconds to wait after no activity before shutting down, default: 5 \ No newline at end of file diff --git a/addOns/client/src/main/resources/org/zaproxy/addon/client/resources/spiderClient-min.yaml b/addOns/client/src/main/resources/org/zaproxy/addon/client/resources/spiderClient-min.yaml new file mode 100644 index 00000000000..79f7261c342 --- /dev/null +++ b/addOns/client/src/main/resources/org/zaproxy/addon/client/resources/spiderClient-min.yaml @@ -0,0 +1,5 @@ + - type: spiderClient # The client spider - a spider which explores modern web apps more effectively + parameters: + context: # String: Name of the context to spider, default: first context + user: # String: An optional user to use for authentication, must be defined in the env + url: # String: URL to start spidering from, default: first context URL diff --git a/addOns/client/src/test/java/org/zaproxy/addon/client/automation/ClientSpiderJobUnitTest.java b/addOns/client/src/test/java/org/zaproxy/addon/client/automation/ClientSpiderJobUnitTest.java new file mode 100644 index 00000000000..25c812576f4 --- /dev/null +++ b/addOns/client/src/test/java/org/zaproxy/addon/client/automation/ClientSpiderJobUnitTest.java @@ -0,0 +1,245 @@ +/* + * Zed Attack Proxy (ZAP) and its related class files. + * + * ZAP is an HTTP/HTTPS proxy for assessing web application security. + * + * Copyright 2024 The ZAP Development Team + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.zaproxy.addon.client.automation; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.nullValue; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.BDDMockito.given; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.withSettings; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import org.apache.commons.httpclient.URIException; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.quality.Strictness; +import org.parosproxy.paros.Constant; +import org.parosproxy.paros.control.Control; +import org.parosproxy.paros.extension.ExtensionLoader; +import org.parosproxy.paros.model.Model; +import org.yaml.snakeyaml.Yaml; +import org.zaproxy.addon.automation.AutomationEnvironment; +import org.zaproxy.addon.automation.AutomationJob.Order; +import org.zaproxy.addon.automation.AutomationProgress; +import org.zaproxy.addon.automation.ContextWrapper; +import org.zaproxy.addon.client.ClientOptions; +import org.zaproxy.addon.client.ExtensionClientIntegration; +import org.zaproxy.addon.client.spider.ClientSpider; +import org.zaproxy.addon.commonlib.Constants; +import org.zaproxy.zap.model.Context; +import org.zaproxy.zap.testutils.TestUtils; +import org.zaproxy.zap.utils.I18N; + +public class ClientSpiderJobUnitTest extends TestUtils { + + private ExtensionLoader extensionLoader; + private ExtensionClientIntegration extClient; + + @BeforeEach + void setUp() { + mockMessages(new ExtensionClientIntegration()); + + extensionLoader = + mock(ExtensionLoader.class, withSettings().strictness(Strictness.LENIENT)); + extClient = mock(ExtensionClientIntegration.class); + given(extensionLoader.getExtension(ExtensionClientIntegration.class)).willReturn(extClient); + + Control.initSingletonForTesting(Model.getSingleton(), extensionLoader); + } + + @Test + void shouldReturnDefaultFieldsAndValues() { + // Given / When + ClientSpiderJob job = new ClientSpiderJob(); + + // Then + assertDefaultJob(job); + assertValidTemplate(job.getTemplateDataMin()); + assertValidTemplate(job.getTemplateDataMax()); + } + + @Test + void shouldVerifyWithoutParameters() { + // Given + AutomationProgress progress = new AutomationProgress(); + ClientSpiderJob job = new ClientSpiderJob(); + job.setJobData(null); + + // When + job.verifyParameters(progress); + + // Then + assertThat(progress.hasWarnings(), is(equalTo(false))); + assertThat(progress.hasErrors(), is(equalTo(false))); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void shouldLoadTemplate(boolean minTemplate) { + // Given + ClientSpiderJob job = new ClientSpiderJob(); + AutomationProgress progress = new AutomationProgress(); + Yaml yaml = new Yaml(); + Object data; + if (minTemplate) { + data = yaml.load(job.getTemplateDataMin()); + } else { + data = yaml.load(job.getTemplateDataMax()); + } + job.setJobData(((LinkedHashMap) ((ArrayList) data).get(0))); + + // When + job.verifyParameters(progress); + + // Then + assertThat(progress.hasErrors(), is(equalTo(false))); + assertThat(progress.hasWarnings(), is(equalTo(false))); + assertDefaultJob(job); + } + + @Test + void shouldSetDefaultParameters() { + // Given + ClientSpiderJob job = new ClientSpiderJob(); + AutomationProgress progress = new AutomationProgress(); + String yamlStr = "parameters:"; + Yaml yaml = new Yaml(); + Object data = yaml.load(yamlStr); + job.setJobData(((LinkedHashMap) data)); + + // When + job.verifyParameters(progress); + + // Then + assertThat(progress.hasErrors(), is(equalTo(false))); + assertThat(progress.hasWarnings(), is(equalTo(false))); + assertThat(job.getParameters().getContext(), is(equalTo(""))); + assertThat(job.getParameters().getUser(), is(equalTo(""))); + assertThat(job.getParameters().getUrl(), is(equalTo(""))); + assertThat(job.getParameters().getMaxDuration(), is(nullValue())); + assertThat(job.getParameters().getMaxCrawlDepth(), is(ClientOptions.DEFAULT_MAX_DEPTH)); + assertThat(job.getParameters().getMaxChildren(), is(nullValue())); + assertThat( + job.getParameters().getNumberOfBrowsers(), is(Constants.getDefaultThreadCount())); + assertThat(job.getParameters().getBrowserId(), is(nullValue())); + assertThat( + job.getParameters().getInitialLoadTime(), + is(ClientOptions.DEFAULT_INITIAL_LOAD_TIME)); + assertThat(job.getParameters().getPageLoadTime(), is(ClientOptions.DEFAULT_PAGE_LOAD_TIME)); + assertThat(job.getParameters().getShutdownTime(), is(ClientOptions.DEFAULT_SHUTDOWN_TIME)); + } + + @Test + void shouldSetParameters() { + // Given + ClientSpiderJob job = new ClientSpiderJob(); + AutomationProgress progress = new AutomationProgress(); + String yamlStr = + """ + parameters: + context: testContext + user: testUser + url: https://www.example.com/test/ + maxDuration: 20 + maxCrawlDepth: 8 + maxChildren: 9 + numberOfBrowsers: 11 + browserId: testBrowser + initialLoadTime: 12 + pageLoadTime: 13 + shutdownTime: 14 + """; + Yaml yaml = new Yaml(); + Object data = yaml.load(yamlStr); + job.setJobData(((LinkedHashMap) data)); + + // When + job.verifyParameters(progress); + + // Then + assertThat(progress.hasErrors(), is(equalTo(false))); + assertThat(progress.hasWarnings(), is(equalTo(false))); + assertThat(job.getParameters().getContext(), is(equalTo("testContext"))); + assertThat(job.getParameters().getUser(), is(equalTo("testUser"))); + assertThat(job.getParameters().getUrl(), is(equalTo("https://www.example.com/test/"))); + assertThat(job.getParameters().getMaxDuration(), is(equalTo(20))); + assertThat(job.getParameters().getMaxCrawlDepth(), is(equalTo(8))); + assertThat(job.getParameters().getMaxChildren(), is(equalTo(9))); + assertThat(job.getParameters().getNumberOfBrowsers(), is(equalTo(11))); + assertThat(job.getParameters().getBrowserId(), is(equalTo("testBrowser"))); + assertThat(job.getParameters().getInitialLoadTime(), is(equalTo(12))); + assertThat(job.getParameters().getPageLoadTime(), is(equalTo(13))); + assertThat(job.getParameters().getShutdownTime(), is(equalTo(14))); + } + + @Test + void shouldRunValidJob() throws URIException, NullPointerException { + // Given + Constant.messages = new I18N(Locale.ENGLISH); + Context context = mock(Context.class); + ContextWrapper contextWrapper = mock(ContextWrapper.class); + given(contextWrapper.getContext()).willReturn(context); + String url = "http://example.com"; + given(contextWrapper.getUrls()).willReturn(List.of(url)); + + ClientSpider clientSpider = mock(ClientSpider.class); + given(extClient.startScan(any(), any(), any(), any(), anyBoolean())).willReturn(1); + given(extClient.getScan(anyInt())).willReturn(clientSpider); + + AutomationProgress progress = new AutomationProgress(); + AutomationEnvironment env = mock(AutomationEnvironment.class); + given(env.replaceVars(url)).willReturn(url); + given(env.getDefaultContextWrapper()).willReturn(contextWrapper); + + ClientSpiderJob job = new ClientSpiderJob(); + + // When + job.runJob(env, progress); + + // Then + assertThat(progress.hasWarnings(), is(equalTo(false))); + assertThat(progress.hasErrors(), is(equalTo(false))); + } + + private static void assertValidTemplate(String value) { + assertThat(value, is(not(equalTo("")))); + assertDoesNotThrow(() -> new Yaml().load(value)); + } + + private static void assertDefaultJob(ClientSpiderJob job) { + assertThat(job.getType(), is(equalTo("spiderClient"))); + assertThat(job.getName(), is(equalTo("spiderClient"))); + assertThat(job.getOrder(), is(equalTo(Order.LAST_EXPLORE))); + assertThat(job.getParamMethodObject(), is(nullValue())); + assertThat(job.getParamMethodName(), is(nullValue())); + } +}