diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2dd85d7c92..99e9bddd32 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,16 +79,24 @@ jobs: key: ${{env.ACTIONS_CACHE_KEY_DATE}} # additional key for cache-busting workspaces: src/agent - name: Linux Prereqs - if: runner.os == 'Linux' && steps.cache-agent-artifacts.outputs.cache-hit != 'true' + if: runner.os == 'Linux' run: | sudo apt-get -y update - sudo apt-get -y install libssl-dev libunwind-dev build-essential pkg-config + sudo apt-get -y install libssl-dev libunwind-dev build-essential pkg-config clang + - name: Clone onefuzz-samples + run: git clone https://github.com/microsoft/onefuzz-samples + - name: Prepare for agent integration tests + shell: bash + working-directory: ./onefuzz-samples/examples/simple-libfuzzer + run: | + make + mkdir -p ../../../src/agent/onefuzz-task/tests/targets/simple + cp fuzz.exe ../../../src/agent/onefuzz-task/tests/targets/simple/fuzz.exe + cp *.pdb ../../../src/agent/onefuzz-task/tests/targets/simple/ 2>/dev/null || : - name: Install Rust Prereqs - if: steps.rust-build-cache.outputs.cache-hit != 'true' && steps.cache-agent-artifacts.outputs.cache-hit != 'true' shell: bash run: src/ci/rust-prereqs.sh - run: src/ci/agent.sh - if: steps.cache-agent-artifacts.outputs.cache-hit != 'true' shell: bash - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 diff --git a/CHANGELOG.md b/CHANGELOG.md index be4779ad77..f02721fa44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,32 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 8.8.0 + +### Added + +* Agent: Added Mariner Linux support for agent VMs [#3306](https://github.com/microsoft/onefuzz/pull/3306) +* Service: Added support for custom ado fields that mark work items as duplicate [#3467](https://github.com/microsoft/onefuzz/pull/3467) +* Service: Permanently store OneFuzz job result data - # crashing input, # regression crashing input, etc. - in Azure storage [#3380](https://github.com/microsoft/onefuzz/pull/3380), [#3439](https://github.com/microsoft/onefuzz/pull/3439) +* Service: Added validation for Iteration/AreaPath on notifications when a job is submitted with a notification config and for `onefuzz debug notification test_template` [#3386](https://github.com/microsoft/onefuzz/pull/3386) + +### Changed + +* Agent: Updated libfuzzer-fuzz basic template to include required args and make it match cli [#3429](https://github.com/microsoft/onefuzz/pull/3429) +* Agent: Downgraded some debug logs from warn to debug [#3450](https://github.com/microsoft/onefuzz/pull/3450) +* CLI: Removed CLI commands from the local fuzzing tasks as they can now be described via yaml template [#3428](https://github.com/microsoft/onefuzz/pull/3428) +* Service: AutoScale table entries are now deleted on VMSS shutdown [#3455](https://github.com/microsoft/onefuzz/pull/3455) + +### Fixed + +* Agent: Fixed local path generation [#3432](https://github.com/microsoft/onefuzz/pull/3432), [#3460](https://github.com/microsoft/onefuzz/pull/3460) + +## 8.7.1 + +### Fixed + +* Service: Removed deprecated Azure retention policy setting that was causing scaleset deployment errors [#3452](https://github.com/microsoft/onefuzz/pull/3452) + ## 8.7.0 ### Added diff --git a/CURRENT_VERSION b/CURRENT_VERSION index c0bcaebe8f..cfc27b4fab 100644 --- a/CURRENT_VERSION +++ b/CURRENT_VERSION @@ -1 +1 @@ -8.7.0 \ No newline at end of file +8.8.0 \ No newline at end of file diff --git a/README.md b/README.md index 010148dd3a..486dae6c15 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,20 @@ # OneFuzz +# IMPORTANT NOTICE + +**_Since September 2020 when OneFuzz was first open sourced, we’ve been on a journey to create a best-in-class orchestrator for running fuzzers, driving security and quality into our products._** + + +**_Initially launched by a small group in MSR, OneFuzz has now become a significant internal platform within Microsoft. As such, we are regretfully archiving the project to focus our attention on becoming a more deeply integrated service within the company. Unfortunately, we aren’t a large enough team to live in both the open-source world and the internal Microsoft world with its own unique set of requirements._** + +**_Our current plan is to archive the project in the next few months. That means we’ll still be making updates for a little while. Of course, even after it’s archived, you’ll still be able to fork it and make the changes you need. Once we’ve decided on a specific date for archiving, we’ll update this readme._** + +**_Thanks for taking the journey with us._** + +**_The OneFuzz team._** + +--- + [![Onefuzz build status](https://github.com/microsoft/onefuzz/workflows/Build/badge.svg?branch=main)](https://github.com/microsoft/onefuzz/actions/workflows/ci.yml?query=branch%3Amain) ## A self-hosted Fuzzing-As-A-Service platform diff --git a/contrib/onefuzz-job-azure-devops-pipeline/ado-work-items.json b/contrib/onefuzz-job-azure-devops-pipeline/ado-work-items.json index eb89fc019d..034d97cf15 100644 --- a/contrib/onefuzz-job-azure-devops-pipeline/ado-work-items.json +++ b/contrib/onefuzz-job-azure-devops-pipeline/ado-work-items.json @@ -13,6 +13,10 @@ "System.AreaPath": "OneFuzz-Ado-Integration", "System.Title": "{{report.task_id}}" }, + "ado_duplicate_fields": { + "System.Reason": "My custom value that means a work item is a duplicate", + "Custom.Work.Item.Field": "My custom value that means a work item is a duplicate" + }, "on_duplicate": { "increment": [], "comment": "DUP {{report.input_sha256}}
Repro Command: 
 {{ repro_cmd }}
", diff --git a/docs/notifications/ado.md b/docs/notifications/ado.md index 131986afba..09dd5b9072 100644 --- a/docs/notifications/ado.md +++ b/docs/notifications/ado.md @@ -51,6 +51,13 @@ clickable, make it a link. "System.Title": "{{ report.crash_site }} - {{ report.executable }}", "Microsoft.VSTS.TCM.ReproSteps": "This is my call stack: " }, + "ado_duplicate_fields": { + "System.Reason": "My custom value that means a work item is a duplicate", + "Custom.Work.Item.Field": "My custom value that means a work item is a duplicate" + // note: the fields and values below are checked by default and don't need to be specified + // "System.Reason": "Duplicate" + // "Microsoft.VSTS.Common.ResolvedReason": "Duplicate" + }, "comment": "This is my comment. {{ report.input_sha256 }} {{ input_url }} 
{{ repro_cmd }}
", "unique_fields": ["System.Title", "System.AreaPath"], "on_duplicate": { diff --git a/src/ApiService/ApiService/OneFuzzTypes/Model.cs b/src/ApiService/ApiService/OneFuzzTypes/Model.cs index b839f52ddc..424669899a 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Model.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Model.cs @@ -689,6 +689,7 @@ public record AdoTemplate( List UniqueFields, Dictionary AdoFields, ADODuplicateTemplate OnDuplicate, + Dictionary? AdoDuplicateFields = null, string? Comment = null ) : NotificationTemplate { public async Task Validate() { @@ -704,8 +705,9 @@ public record RenderedAdoTemplate( List UniqueFields, Dictionary AdoFields, ADODuplicateTemplate OnDuplicate, + Dictionary? AdoDuplicateFields = null, string? Comment = null - ) : AdoTemplate(BaseUrl, AuthToken, Project, Type, UniqueFields, AdoFields, OnDuplicate, Comment); + ) : AdoTemplate(BaseUrl, AuthToken, Project, Type, UniqueFields, AdoFields, OnDuplicate, AdoDuplicateFields, Comment); public record TeamsTemplate(SecretData Url) : NotificationTemplate { public Task Validate() { diff --git a/src/ApiService/ApiService/onefuzzlib/notifications/Ado.cs b/src/ApiService/ApiService/onefuzzlib/notifications/Ado.cs index e05bb9bc24..3780bc1b2b 100644 --- a/src/ApiService/ApiService/onefuzzlib/notifications/Ado.cs +++ b/src/ApiService/ApiService/onefuzzlib/notifications/Ado.cs @@ -291,6 +291,7 @@ public static RenderedAdoTemplate RenderAdoTemplate(ILogger logTracer, Renderer original.UniqueFields, adoFields, onDuplicate, + original.AdoDuplicateFields, original.Comment != null ? Render(renderer, original.Comment, instanceUrl, logTracer) : null ); } @@ -535,7 +536,7 @@ public async Async.Task Process(IList<(string, string)> notificationInfo) { _logTracer.AddTags(new List<(string, string)> { ("MatchingWorkItemIds", $"{workItem.Id}") }); _logTracer.LogInformation("Found matching work item"); } - if (IsADODuplicateWorkItem(workItem)) { + if (IsADODuplicateWorkItem(workItem, _config.AdoDuplicateFields)) { continue; } @@ -575,13 +576,17 @@ public async Async.Task Process(IList<(string, string)> notificationInfo) { } } - private static bool IsADODuplicateWorkItem(WorkItem wi) { + private static bool IsADODuplicateWorkItem(WorkItem wi, Dictionary? duplicateFields) { // A work item could have System.State == Resolve && System.Reason == Duplicate // OR it could have System.State == Closed && System.Reason == Duplicate // I haven't found any other combinations where System.Reason could be duplicate but just to be safe // we're explicitly _not_ checking the state of the work item to determine if it's duplicate return wi.Fields.ContainsKey("System.Reason") && string.Equals(wi.Fields["System.Reason"].ToString(), "Duplicate", StringComparison.OrdinalIgnoreCase) || wi.Fields.ContainsKey("Microsoft.VSTS.Common.ResolvedReason") && string.Equals(wi.Fields["Microsoft.VSTS.Common.ResolvedReason"].ToString(), "Duplicate", StringComparison.OrdinalIgnoreCase) + || duplicateFields?.Any(fieldPair => { + var (field, value) = fieldPair; + return wi.Fields.ContainsKey(field) && string.Equals(wi.Fields[field].ToString(), value, StringComparison.OrdinalIgnoreCase); + }) == true // Alternatively, the work item can also specify a 'relation' to another work item. // This is typically used to create parent/child relationships between work items but can also // Be used to mark duplicates so we should check this as well. diff --git a/src/ApiService/IntegrationTests/JinjaToScribanMigrationTests.cs b/src/ApiService/IntegrationTests/JinjaToScribanMigrationTests.cs index 0ae3b11cb5..4033a05369 100644 --- a/src/ApiService/IntegrationTests/JinjaToScribanMigrationTests.cs +++ b/src/ApiService/IntegrationTests/JinjaToScribanMigrationTests.cs @@ -111,6 +111,7 @@ public async Async.Task OptionalFieldsAreSupported() { }, "{{ if org }} blah {{ end }}" ), + null, "{{ if org }} blah {{ end }}" ); @@ -137,6 +138,7 @@ public async Async.Task All_ADO_Fields_Are_Migrated() { }, "{% if org %} comment {% endif %}" ), + null, "{% if org %} comment {% endif %}" ); diff --git a/src/ApiService/Tests/OrmModelsTest.cs b/src/ApiService/Tests/OrmModelsTest.cs index 1aa7d2d163..956d0c30c5 100644 --- a/src/ApiService/Tests/OrmModelsTest.cs +++ b/src/ApiService/Tests/OrmModelsTest.cs @@ -232,6 +232,7 @@ from authToken in Arb.Generate>() from str in Arb.Generate() from fields in Arb.Generate>() from adoFields in Arb.Generate>() + from adoDuplicateFields in Arb.Generate>() from dupeTemplate in Arb.Generate() select new AdoTemplate( baseUrl, @@ -241,6 +242,7 @@ from dupeTemplate in Arb.Generate() fields, adoFields, dupeTemplate, + adoDuplicateFields, str.Get)); public static Arbitrary ArbTeamsTemplate() diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock index 254684be97..f395e93408 100644 --- a/src/agent/Cargo.lock +++ b/src/agent/Cargo.lock @@ -14,7 +14,7 @@ version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" dependencies = [ - "gimli", + "gimli 0.27.3", ] [[package]] @@ -60,16 +60,15 @@ dependencies = [ [[package]] name = "anstream" -version = "0.3.2" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" +checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c" dependencies = [ "anstyle", "anstyle-parse", "anstyle-query", "anstyle-wincon", "colorchoice", - "is-terminal", "utf8parse", ] @@ -99,9 +98,9 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "1.0.2" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c677ab05e09154296dd37acecd46420c17b9713e8366facafa8fc0885167cf4c" +checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd" dependencies = [ "anstyle", "windows-sys 0.48.0", @@ -200,9 +199,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "azure_core" -version = "0.13.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b0f0eea648347e40f5f7f7e6bfea4553bcefad0fbf52044ea339e5ce3aba61" +checksum = "2331555a3618a32516c6172a63e9fec4af0edb43c6fcfeb5303a0716fc34498b" dependencies = [ "async-trait", "base64 0.21.2", @@ -214,7 +213,7 @@ dependencies = [ "log", "paste", "pin-project", - "quick-xml 0.29.0", + "quick-xml 0.30.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.8.5", "reqwest", "rustc_version", @@ -227,9 +226,9 @@ dependencies = [ [[package]] name = "azure_storage" -version = "0.13.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32d9cfa13ed9acb51cd663e04f343bd550a92b455add96c90de387a9a6bc4dbc" +checksum = "16565073e533053f4e29e6b139de2af758e984108a1cddbb1a432387e7f4474d" dependencies = [ "RustyXML", "async-trait", @@ -250,9 +249,9 @@ dependencies = [ [[package]] name = "azure_storage_blobs" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57cb0fe58af32a3fb49e560613cb1e4937f9f13161a2c1caf1bba0224435f2af" +checksum = "0900e63940d1ba51039efda3d8cf658157a1c75449081a6e18069d2588809329" dependencies = [ "RustyXML", "azure_core", @@ -397,9 +396,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" dependencies = [ "serde", ] @@ -465,33 +464,31 @@ dependencies = [ [[package]] name = "clap" -version = "4.3.21" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c27cdf28c0f604ba3f512b0c9a409f8de8513e4816705deb0498b627e7c3a3fd" +checksum = "6a13b88d2c62ff462f88e4a121f17a82c1af05693a2f192b5c38d14de73c19f6" dependencies = [ "clap_builder", "clap_derive", - "once_cell", ] [[package]] name = "clap_builder" -version = "4.3.21" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08a9f1ab5e9f01a9b81f202e8562eb9a10de70abf9eaeac1be465c28b75aa4aa" +checksum = "2bb9faaa7c2ef94b2743a21f5a29e6f0010dff4caa69ac8e9d6cf8b6fa74da08" dependencies = [ "anstream", "anstyle", "clap_lex", - "once_cell", "strsim", ] [[package]] name = "clap_derive" -version = "4.3.12" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54a9bb5758fc5dfe728d1019941681eccaf0cf8a4189b692a0ee2f2ecf90a050" +checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873" dependencies = [ "heck", "proc-macro2 1.0.66", @@ -510,7 +507,9 @@ name = "cobertura" version = "0.1.0" dependencies = [ "anyhow", - "quick-xml 0.30.0", + "async-trait", + "quick-xml 0.30.0 (git+https://github.com/tafia/quick-xml/)", + "tokio", ] [[package]] @@ -580,6 +579,7 @@ dependencies = [ "symbolic", "tempfile", "thiserror", + "tokio", ] [[package]] @@ -749,7 +749,7 @@ dependencies = [ "anyhow", "clap", "elsa", - "gimli", + "gimli 0.28.0", "goblin", "iced-x86", "log", @@ -863,7 +863,7 @@ dependencies = [ "regex", "thiserror", "windows", - "winreg 0.50.0", + "winreg 0.51.0", ] [[package]] @@ -883,9 +883,9 @@ dependencies = [ [[package]] name = "elsa" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e0aca8dce8856e420195bd13b6a64de3334235ccc9214e824b86b12bf26283" +checksum = "714f766f3556b44e7e4776ad133fcc3445a489517c25c704ace411bb14790194" dependencies = [ "stable_deref_trait", ] @@ -968,6 +968,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + [[package]] name = "fastrand" version = "1.9.0" @@ -1257,8 +1263,18 @@ version = "0.27.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" dependencies = [ - "fallible-iterator", - "indexmap 1.9.3", + "fallible-iterator 0.2.0", + "stable_deref_trait", +] + +[[package]] +name = "gimli" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" +dependencies = [ + "fallible-iterator 0.3.0", + "indexmap 2.0.0", "stable_deref_trait", ] @@ -2035,18 +2051,19 @@ dependencies = [ [[package]] name = "notify" -version = "6.0.1" +version = "6.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5738a2795d57ea20abec2d6d76c6081186709c0024187cd5977265eda6598b51" +checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.3.3", "filetime", "inotify", "kqueue", "libc", + "log", "mio 0.8.8", "walkdir", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -2151,7 +2168,7 @@ dependencies = [ "urlparse", "uuid", "windows", - "winreg 0.50.0", + "winreg 0.51.0", ] [[package]] @@ -2193,7 +2210,7 @@ dependencies = [ "coverage", "debuggable-module", "pretty_assertions", - "quick-xml 0.30.0", + "quick-xml 0.30.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde", "serde_json", ] @@ -2403,7 +2420,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "82040a392923abe6279c00ab4aff62d5250d1c8555dc780e4b02783a7aa74863" dependencies = [ - "fallible-iterator", + "fallible-iterator 0.2.0", "scroll", "uuid", ] @@ -2436,9 +2453,9 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pete" -version = "0.10.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "229eb6b3cb0d3d075727c614687ab08384cac3b75fa100e1e08b30d7bee39d00" +checksum = "0f09c1c1ad40df294ff8643fe88a3dc64fff3293b6bc0ed9f71aff71f7086cbd" dependencies = [ "libc", "memoffset 0.8.0", @@ -2589,9 +2606,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.29.0" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81b9228215d82c7b61490fec1de287136b5de6f5700f6e58ea9ad61a7964ca51" +checksum = "eff6510e86862b57b210fd8cbe8ed3f0d7d600b9c2863cd4549a2e033c66e956" dependencies = [ "memchr", "serde", @@ -2600,11 +2617,10 @@ dependencies = [ [[package]] name = "quick-xml" version = "0.30.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eff6510e86862b57b210fd8cbe8ed3f0d7d600b9c2863cd4549a2e033c66e956" +source = "git+https://github.com/tafia/quick-xml/#efb2dc00eba4b0502b10fb481225b13dab41447e" dependencies = [ "memchr", - "serde", + "tokio", ] [[package]] @@ -3278,7 +3294,7 @@ dependencies = [ "flume", "num_cpus", "queue-file", - "quick-xml 0.30.0", + "quick-xml 0.30.0 (registry+https://github.com/rust-lang/crates.io-index)", "regex", "reqwest", "reqwest-retry", @@ -3368,9 +3384,9 @@ dependencies = [ "dmsort", "elementtree", "elsa", - "fallible-iterator", + "fallible-iterator 0.2.0", "flate2", - "gimli", + "gimli 0.27.3", "goblin", "lazy_static", "nom", @@ -3468,9 +3484,9 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.7.1" +version = "3.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc02fddf48964c42031a0b3fe0428320ecf3a73c401040fc0096f97794310651" +checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" dependencies = [ "cfg-if 1.0.0", "fastrand 2.0.0", @@ -3555,9 +3571,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.30.0" +version = "1.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d3ce25f50619af8b0aec2eb23deebe84249e19e2ddd393a6e16e3300a6dadfd" +checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" dependencies = [ "backtrace", "bytes", @@ -3950,7 +3966,7 @@ dependencies = [ "os_pipe", "tempfile", "windows", - "winreg 0.50.0", + "winreg 0.51.0", ] [[package]] @@ -4148,9 +4164,9 @@ dependencies = [ [[package]] name = "winreg" -version = "0.50.0" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +checksum = "937f3df7948156640f46aacef17a70db0de5917bda9c92b0f751f3a955b588fc" dependencies = [ "cfg-if 1.0.0", "windows-sys 0.48.0", diff --git a/src/agent/cobertura/Cargo.toml b/src/agent/cobertura/Cargo.toml index be57083057..ea121ad1b9 100644 --- a/src/agent/cobertura/Cargo.toml +++ b/src/agent/cobertura/Cargo.toml @@ -6,4 +6,8 @@ license = "MIT" [dependencies] anyhow = "1.0" -quick-xml = "0.30" +quick-xml = { git = "https://github.com/tafia/quick-xml/", features = [ + "async-tokio", +] } +tokio = "1" +async-trait = "0.1" diff --git a/src/agent/cobertura/src/lib.rs b/src/agent/cobertura/src/lib.rs index e919e08733..328617344d 100644 --- a/src/agent/cobertura/src/lib.rs +++ b/src/agent/cobertura/src/lib.rs @@ -3,7 +3,9 @@ use std::io::{Cursor, Write}; +use async_trait::async_trait; use quick_xml::{Result, Writer}; +use tokio::io::AsyncWrite; impl CoberturaCoverage { pub fn to_string(&self) -> anyhow::Result { @@ -28,6 +30,18 @@ pub trait WriteXml { } } +#[async_trait] +pub trait WriteXmlAsync { + async fn _write_xml_async( + &self, + writer: &mut Writer, + ) -> Result<()>; + async fn write_xml_async(&self, writer: W) -> Result<()> { + let mut writer = Writer::new(writer); + self._write_xml_async(&mut writer).await + } +} + // Only write optional fields if present. impl WriteXml for Option where @@ -42,6 +56,23 @@ where } } +#[async_trait] +impl WriteXmlAsync for Option +where + T: WriteXmlAsync + Sync + Send + Unpin, +{ + async fn _write_xml_async( + &self, + writer: &mut Writer, + ) -> Result<()> { + if let Some(value) = self { + value._write_xml_async(writer).await?; + } + + Ok(()) + } +} + impl WriteXml for Vec where T: WriteXml, @@ -55,6 +86,23 @@ where } } +#[async_trait] +impl WriteXmlAsync for Vec +where + T: WriteXmlAsync + Sync + Send + Unpin, +{ + async fn _write_xml_async( + &self, + writer: &mut Writer, + ) -> Result<()> { + for value in self { + value._write_xml_async(writer).await?; + } + + Ok(()) + } +} + macro_rules! float { ($val: expr) => { format!("{:.02}", $val).as_str() @@ -131,6 +179,37 @@ impl WriteXml for CoberturaCoverage { } } +#[async_trait] +impl WriteXmlAsync for CoberturaCoverage { + async fn _write_xml_async( + &self, + writer: &mut Writer, + ) -> Result<()> { + writer + .create_element("coverage") + .with_attributes([ + ("line-rate", float!(self.line_rate)), + ("branch-rate", float!(self.branch_rate)), + ("lines-covered", uint!(self.lines_covered)), + ("lines-valid", uint!(self.lines_valid)), + ("branches-covered", uint!(self.branches_covered)), + ("branches-valid", uint!(self.branches_valid)), + ("complexity", uint!(self.complexity)), + ("version", string!(self.version)), + ("timestamp", uint!(self.timestamp)), + ]) + .write_inner_content_async(|w| async { + self.sources._write_xml_async(w).await?; + self.packages._write_xml_async(w).await?; + + Ok(w) + }) + .await?; + + Ok(()) + } +} + // #[derive(Clone, Debug, Default)] pub struct Sources { @@ -147,6 +226,24 @@ impl WriteXml for Sources { } } +#[async_trait] +impl WriteXmlAsync for Sources { + async fn _write_xml_async( + &self, + writer: &mut Writer, + ) -> Result<()> { + writer + .create_element("sources") + .write_inner_content_async(|w| async { + self.sources._write_xml_async(w).await?; + Ok(w) + }) + .await?; + + Ok(()) + } +} + // #[derive(Clone, Debug, Default)] pub struct Source { @@ -164,6 +261,22 @@ impl WriteXml for Source { } } +#[async_trait] +impl WriteXmlAsync for Source { + async fn _write_xml_async( + &self, + writer: &mut Writer, + ) -> Result<()> { + writer + .create_element("source") + .with_attributes([("path", string!(self.path))]) + .write_empty_async() + .await?; + + Ok(()) + } +} + // #[derive(Clone, Debug, Default)] pub struct Packages { @@ -180,6 +293,24 @@ impl WriteXml for Packages { } } +#[async_trait] +impl WriteXmlAsync for Packages { + async fn _write_xml_async( + &self, + writer: &mut Writer, + ) -> Result<()> { + writer + .create_element("packages") + .write_inner_content_async(|w| async { + self.packages._write_xml_async(w).await?; + Ok(w) + }) + .await?; + + Ok(()) + } +} + // // // @@ -211,6 +342,30 @@ impl WriteXml for Package { } } +#[async_trait] +impl WriteXmlAsync for Package { + async fn _write_xml_async( + &self, + writer: &mut Writer, + ) -> Result<()> { + writer + .create_element("package") + .with_attributes([ + ("name", string!(self.name)), + ("line-rate", float!(self.line_rate)), + ("branch-rate", float!(self.branch_rate)), + ("complexity", uint!(self.complexity)), + ]) + .write_inner_content_async(|w| async { + self.classes._write_xml_async(w).await?; + Ok(w) + }) + .await?; + + Ok(()) + } +} + // #[derive(Clone, Debug, Default)] pub struct Classes { @@ -227,6 +382,24 @@ impl WriteXml for Classes { } } +#[async_trait] +impl WriteXmlAsync for Classes { + async fn _write_xml_async( + &self, + writer: &mut Writer, + ) -> Result<()> { + writer + .create_element("classes") + .write_inner_content_async(|w| async { + self.classes._write_xml_async(w).await?; + Ok(w) + }) + .await?; + + Ok(()) + } +} + // // // @@ -266,6 +439,32 @@ impl WriteXml for Class { } } +#[async_trait] +impl WriteXmlAsync for Class { + async fn _write_xml_async( + &self, + writer: &mut Writer, + ) -> Result<()> { + writer + .create_element("class") + .with_attributes([ + ("name", string!(self.name)), + ("filename", string!(self.filename)), + ("line-rate", float!(self.line_rate)), + ("branch-rate", float!(self.branch_rate)), + ("complexity", uint!(self.complexity)), + ]) + .write_inner_content_async(|w| async { + self.methods._write_xml_async(w).await?; + self.lines._write_xml_async(w).await?; + Ok(w) + }) + .await?; + + Ok(()) + } +} + // #[derive(Clone, Debug, Default)] pub struct Methods { @@ -282,6 +481,24 @@ impl WriteXml for Methods { } } +#[async_trait] +impl WriteXmlAsync for Methods { + async fn _write_xml_async( + &self, + writer: &mut Writer, + ) -> Result<()> { + writer + .create_element("methods") + .write_inner_content_async(|w| async { + self.methods._write_xml_async(w).await?; + Ok(w) + }) + .await?; + + Ok(()) + } +} + // // // @@ -313,6 +530,30 @@ impl WriteXml for Method { } } +#[async_trait] +impl WriteXmlAsync for Method { + async fn _write_xml_async( + &self, + writer: &mut Writer, + ) -> Result<()> { + writer + .create_element("method") + .with_attributes([ + ("name", string!(self.name)), + ("signature", string!(self.signature)), + ("line-rate", float!(self.line_rate)), + ("branch-rate", float!(self.branch_rate)), + ]) + .write_inner_content_async(|w| async { + self.lines._write_xml_async(w).await?; + Ok(w) + }) + .await?; + + Ok(()) + } +} + // #[derive(Clone, Debug, Default)] pub struct Lines { @@ -329,6 +570,24 @@ impl WriteXml for Lines { } } +#[async_trait] +impl WriteXmlAsync for Lines { + async fn _write_xml_async( + &self, + writer: &mut Writer, + ) -> Result<()> { + writer + .create_element("lines") + .write_inner_content_async(|w| async { + self.lines._write_xml_async(w).await?; + Ok(w) + }) + .await?; + + Ok(()) + } +} + // // // @@ -366,6 +625,36 @@ impl WriteXml for Line { } } +#[async_trait] +impl WriteXmlAsync for Line { + async fn _write_xml_async( + &self, + writer: &mut Writer, + ) -> Result<()> { + let condition_coverage = if let Some(s) = &self.condition_coverage { + s.as_str() + } else { + "100%" + }; + + writer + .create_element("line") + .with_attributes([ + ("number", uint!(self.number)), + ("hits", uint!(self.hits)), + ("branch", boolean!(self.branch.unwrap_or_default())), + ("condition-coverage", condition_coverage), + ]) + .write_inner_content_async(|w| async { + self.conditions._write_xml_async(w).await?; + Ok(w) + }) + .await?; + + Ok(()) + } +} + // #[derive(Clone, Debug, Default)] pub struct Conditions { @@ -382,6 +671,23 @@ impl WriteXml for Conditions { } } +#[async_trait] +impl WriteXmlAsync for Conditions { + async fn _write_xml_async( + &self, + writer: &mut Writer, + ) -> Result<()> { + writer + .create_element("conditions") + .write_inner_content_async(|w| async { + self.conditions._write_xml_async(w).await?; + Ok(w) + }) + .await?; + + Ok(()) + } +} // // // @@ -407,3 +713,23 @@ impl WriteXml for Condition { Ok(()) } } + +#[async_trait] +impl WriteXmlAsync for Condition { + async fn _write_xml_async( + &self, + writer: &mut Writer, + ) -> Result<()> { + writer + .create_element("condition") + .with_attributes([ + ("number", uint!(self.number)), + ("type", uint!(self.r#type)), + ("coverage", uint!(self.coverage)), + ]) + .write_empty_async() + .await?; + + Ok(()) + } +} diff --git a/src/agent/coverage/Cargo.toml b/src/agent/coverage/Cargo.toml index 29e67523d9..2da324de08 100644 --- a/src/agent/coverage/Cargo.toml +++ b/src/agent/coverage/Cargo.toml @@ -26,17 +26,18 @@ debugger = { path = "../debugger" } [target.'cfg(target_os = "linux")'.dependencies] nix = "0.26" -pete = "0.10" +pete = "0.12" # For procfs, opt out of the `chrono` freature; it pulls in an old version # of `time`. We do not use the methods that the `chrono` feature enables. procfs = { version = "0.15.1", default-features = false, features = ["flate2"] } [dev-dependencies] -clap = { version = "4.3", features = ["derive"] } +clap = { version = "4.4", features = ["derive"] } env_logger = "0.10.0" pretty_assertions = "1.4.0" insta = { version = "1.31.0", features = ["glob"] } coverage = { path = "../coverage" } cc = "1.0" -tempfile = "3.7.0" +tempfile = "3.8.0" dunce = "1.0" +tokio = { version = "1", features = ["full"] } diff --git a/src/agent/coverage/examples/cobertura.rs b/src/agent/coverage/examples/cobertura.rs index 2ba5ec2f6a..c783223175 100644 --- a/src/agent/coverage/examples/cobertura.rs +++ b/src/agent/coverage/examples/cobertura.rs @@ -8,11 +8,11 @@ use coverage::source::{Count, FileCoverage, Line, SourceCoverage}; use debuggable_module::path::FilePath; fn main() -> Result<()> { - println!("{}", generate_output()?); + println!("{}", generate_output()?.to_string()?); Ok(()) } -fn generate_output() -> Result { +fn generate_output() -> Result { let modoff = vec![ (r"/missing/lib.c", vec![1, 2, 3, 5, 8]), ( @@ -40,19 +40,24 @@ fn generate_output() -> Result { coverage.files.insert(file_path, file); } - CoberturaCoverage::from(&coverage).to_string() + Ok(CoberturaCoverage::from(&coverage)) } #[cfg(test)] mod test { + use std::io::Write; + use super::*; + use cobertura::WriteXml; + use cobertura::WriteXmlAsync; use pretty_assertions::assert_eq; + use tokio::{self, io::AsyncWriteExt}; #[test] // On Windows this produces different output due to filename parsing. #[cfg(target_os = "linux")] pub fn check_output() { - let result = generate_output().unwrap(); + let result = generate_output().unwrap().to_string().unwrap(); let expected = r#" @@ -223,4 +228,28 @@ mod test { assert_eq!(expected, result); } + + #[tokio::test] + async fn sync_and_async_are_identical() { + let mut sync_data = Vec::new(); + let mut sync_writer = std::io::BufWriter::new(&mut sync_data); + generate_output() + .unwrap() + .write_xml(&mut sync_writer) + .unwrap(); + sync_writer.flush(); + + let mut async_data = Vec::new(); + let mut async_writer = tokio::io::BufWriter::new(&mut async_data); + generate_output() + .unwrap() + .write_xml_async(&mut async_writer) + .await + .unwrap(); + async_writer.flush().await; + + drop(sync_writer); + drop(async_writer); + assert_eq!(&sync_data, &async_data); + } } diff --git a/src/agent/coverage/src/record/linux/debugger.rs b/src/agent/coverage/src/record/linux/debugger.rs index e2502e8d2e..c4512c4fbb 100644 --- a/src/agent/coverage/src/record/linux/debugger.rs +++ b/src/agent/coverage/src/record/linux/debugger.rs @@ -4,6 +4,7 @@ use std::collections::BTreeMap; use std::io::Read; use std::process::{Child, Command}; +use std::time::Duration; use anyhow::{bail, format_err, Result}; use debuggable_module::path::FilePath; @@ -75,7 +76,11 @@ impl<'eh> Debugger<'eh> { // These calls should also be unnecessary no-ops, but we really want to avoid any dangling // or zombie child processes. let _ = child.kill(); - let _ = child.wait(); + + // We don't need to call child.wait() because of the following series of events: + // 1. pete, our ptracing library, spawns the child process with ptrace flags + // 2. rust stdlib set SIG_IGN as the SIGCHLD handler: https://github.com/rust-lang/rust/issues/110317 + // 3. linux kernel automatically reaps pids when the above 2 hold: https://github.com/torvalds/linux/blob/44149752e9987a9eac5ad78e6d3a20934b5e018d/kernel/signal.c#L2089-L2110 let output = Output { status, @@ -198,8 +203,8 @@ impl DebuggerContext { pub fn new() -> Self { let breakpoints = Breakpoints::default(); let images = None; - let tracer = Ptracer::new(); - + let mut tracer = Ptracer::new(); + *tracer.poll_delay_mut() = Duration::from_millis(1); Self { breakpoints, images, diff --git a/src/agent/debuggable-module/Cargo.toml b/src/agent/debuggable-module/Cargo.toml index 1cd11dfd30..a227432830 100644 --- a/src/agent/debuggable-module/Cargo.toml +++ b/src/agent/debuggable-module/Cargo.toml @@ -6,8 +6,8 @@ license = "MIT" [dependencies] anyhow = "1.0" -elsa = "1.8.1" -gimli = "0.27.2" +elsa = "1.9.0" +gimli = "0.28.0" goblin = "0.6" iced-x86 = "1.20" log = "0.4.17" @@ -21,4 +21,4 @@ symbolic = { version = "12.3", features = [ thiserror = "1.0" [dev-dependencies] -clap = { version = "4.3", features = ["derive"] } +clap = { version = "4.4", features = ["derive"] } diff --git a/src/agent/dynamic-library/Cargo.toml b/src/agent/dynamic-library/Cargo.toml index 604d221700..ad3cc482d2 100644 --- a/src/agent/dynamic-library/Cargo.toml +++ b/src/agent/dynamic-library/Cargo.toml @@ -6,14 +6,14 @@ license = "MIT" [dependencies] anyhow = "1.0" -clap = { version = "4.3.0", features = ["derive"] } +clap = { version = "4.4.2", features = ["derive"] } lazy_static = "1.4" regex = "1.9" thiserror = "1.0" [target.'cfg(windows)'.dependencies] debugger = { path = "../debugger" } -winreg = "0.50" +winreg = "0.51" [dependencies.windows] version = "0.48" diff --git a/src/agent/onefuzz-agent/Cargo.toml b/src/agent/onefuzz-agent/Cargo.toml index 5ce8669766..bc73d37973 100644 --- a/src/agent/onefuzz-agent/Cargo.toml +++ b/src/agent/onefuzz-agent/Cargo.toml @@ -22,7 +22,7 @@ reqwest = { version = "0.11", features = [ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" storage-queue = { path = "../storage-queue" } -tokio = { version = "1.29", features = ["full"] } +tokio = { version = "1.32", features = ["full"] } url = { version = "2.4", features = ["serde"] } uuid = { version = "1.4", features = ["serde", "v4"] } clap = { version = "4", features = ["derive", "cargo"] } @@ -31,13 +31,13 @@ onefuzz-telemetry = { path = "../onefuzz-telemetry" } backtrace = "0.3" ipc-channel = { git = "https://github.com/servo/ipc-channel", rev = "7f432aa" } dynamic-library = { path = "../dynamic-library" } -azure_core = { version = "0.13", default-features = false, features = [ +azure_core = { version = "0.14", default-features = false, features = [ "enable_reqwest", ] } -azure_storage = { version = "0.13", default-features = false, features = [ +azure_storage = { version = "0.14", default-features = false, features = [ "enable_reqwest", ] } -azure_storage_blobs = { version = "0.13", default-features = false, features = [ +azure_storage_blobs = { version = "0.14", default-features = false, features = [ "enable_reqwest", ] } diff --git a/src/agent/onefuzz-task/Cargo.toml b/src/agent/onefuzz-task/Cargo.toml index 4e0bd381b0..d5588a58e6 100644 --- a/src/agent/onefuzz-task/Cargo.toml +++ b/src/agent/onefuzz-task/Cargo.toml @@ -6,6 +6,14 @@ edition = "2021" publish = false license = "MIT" +[lib] +path = "src/lib.rs" +name = "onefuzz_task_lib" + +[[bin]] +path = "src/main.rs" +name = "onefuzz-task" + [features] integration_test = [] @@ -46,9 +54,9 @@ strum = "0.25" strum_macros = "0.25" stacktrace-parser = { path = "../stacktrace-parser" } storage-queue = { path = "../storage-queue" } -tempfile = "3.7.0" +tempfile = "3.8.0" thiserror = "1.0" -tokio = { version = "1.29", features = ["full"] } +tokio = { version = "1.32", features = ["full"] } tokio-util = { version = "0.7", features = ["full"] } tokio-stream = "0.1" tui = { package = "ratatui", version = "0.22.0", default-features = false, features = [ @@ -62,13 +70,13 @@ chrono = { version = "0.4", default-features = false, features = [ ] } ipc-channel = { git = "https://github.com/servo/ipc-channel", rev = "7f432aa" } -azure_core = { version = "0.13", default-features = false, features = [ +azure_core = { version = "0.14", default-features = false, features = [ "enable_reqwest", ] } -azure_storage = { version = "0.13", default-features = false, features = [ +azure_storage = { version = "0.14", default-features = false, features = [ "enable_reqwest", ] } -azure_storage_blobs = { version = "0.13", default-features = false, features = [ +azure_storage_blobs = { version = "0.14", default-features = false, features = [ "enable_reqwest", ] } @@ -77,3 +85,4 @@ schemars = { version = "0.8.12", features = ["uuid1"] } [dev-dependencies] pretty_assertions = "1.4" +tempfile = "3.8" diff --git a/src/agent/onefuzz-task/src/lib.rs b/src/agent/onefuzz-task/src/lib.rs new file mode 100644 index 0000000000..997eea549d --- /dev/null +++ b/src/agent/onefuzz-task/src/lib.rs @@ -0,0 +1,9 @@ +#[macro_use] +extern crate anyhow; +#[macro_use] +extern crate clap; +#[macro_use] +extern crate onefuzz_telemetry; + +pub mod local; +pub mod tasks; diff --git a/src/agent/onefuzz-task/src/tasks/coverage/generic.rs b/src/agent/onefuzz-task/src/tasks/coverage/generic.rs index 4fde9efb31..a915271bb8 100644 --- a/src/agent/onefuzz-task/src/tasks/coverage/generic.rs +++ b/src/agent/onefuzz-task/src/tasks/coverage/generic.rs @@ -10,7 +10,7 @@ use std::time::Duration; use anyhow::{bail, Context, Result}; use async_trait::async_trait; -use cobertura::{CoberturaCoverage, WriteXml}; +use cobertura::{CoberturaCoverage, WriteXmlAsync}; use coverage::allowlist::AllowList; use coverage::binary::{BinaryCoverage, DebugInfoCache}; use coverage::record::CoverageRecorder; @@ -161,7 +161,7 @@ impl CoverageTask { } if seen_inputs { - context.report_coverage_stats().await?; + context.report_coverage_stats().await; context.save_and_sync_coverage().await?; } @@ -454,7 +454,7 @@ impl<'a> TaskContext<'a> { Ok(count) } - pub async fn report_coverage_stats(&self) -> Result<()> { + pub async fn report_coverage_stats(&self) { use EventData::*; let coverage = RwLock::read(&self.coverage).await; @@ -471,7 +471,6 @@ impl<'a> TaskContext<'a> { ]), ) .await; - Ok(()) } pub async fn save_coverage( @@ -525,11 +524,13 @@ impl<'a> TaskContext<'a> { async fn save_cobertura_xml(source: &SourceCoverage, path: &Path) -> Result<(), anyhow::Error> { let cobertura = CoberturaCoverage::from(source); - let cobertura_coverage_file = std::fs::File::create(path) + let cobertura_coverage_file = tokio::fs::File::create(path) + .await .with_context(|| format!("creating cobertura coverage file {}", path.display()))?; - let cobertura_coverage_file_writer = std::io::BufWriter::new(cobertura_coverage_file); + let cobertura_coverage_file_writer = tokio::io::BufWriter::new(cobertura_coverage_file); cobertura - .write_xml(cobertura_coverage_file_writer) + .write_xml_async(cobertura_coverage_file_writer) + .await .with_context(|| format!("serializing cobertura coverage to {}", path.display()))?; Ok(()) } @@ -565,7 +566,7 @@ impl<'a> Processor for TaskContext<'a> { self.heartbeat.alive(); self.record_input(input).await?; - self.report_coverage_stats().await?; + self.report_coverage_stats().await; self.save_and_sync_coverage().await?; Ok(()) diff --git a/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/common.rs b/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/common.rs index bfd9f3f5cc..32f3372958 100644 --- a/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/common.rs +++ b/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/common.rs @@ -272,7 +272,7 @@ where info!("config is: {:?}", self.config); let fuzzer = L::from_config(&self.config).await?; - let mut running = fuzzer.fuzz(crash_dir.path(), local_inputs, &inputs).await?; + let mut running = fuzzer.fuzz(crash_dir.path(), local_inputs, &inputs)?; info!("child is: {:?}", running); diff --git a/src/agent/onefuzz-task/tests/template_integration.rs b/src/agent/onefuzz-task/tests/template_integration.rs new file mode 100644 index 0000000000..d0e68e5d02 --- /dev/null +++ b/src/agent/onefuzz-task/tests/template_integration.rs @@ -0,0 +1,212 @@ +use std::{ + collections::HashSet, + ffi::OsStr, + path::{Path, PathBuf}, +}; + +use tokio::fs; + +use anyhow::Result; +use log::info; +use onefuzz_task_lib::local::template; +use std::time::Duration; +use tokio::time::timeout; + +macro_rules! libfuzzer_tests { + ($($name:ident: $value:expr,)*) => { + $( + #[tokio::test(flavor = "multi_thread")] + #[cfg_attr(not(feature = "integration_test"), ignore)] + async fn $name() { + let _ = env_logger::builder().is_test(true).try_init(); + let (config, libfuzzer_target) = $value; + test_libfuzzer_basic_template(PathBuf::from(config), PathBuf::from(libfuzzer_target)).await; + } + )* + } +} + +// This is the format for adding other templates/targets for this macro +// $TEST_NAME: ($RELATIVE_PATH_TO_TEMPLATE, $RELATIVE_PATH_TO_TARGET), +// Make sure that you place the target binary in CI +libfuzzer_tests! { + libfuzzer_basic: ("./tests/templates/libfuzzer_basic.yml", "./tests/targets/simple/fuzz.exe"), +} + +async fn test_libfuzzer_basic_template(config: PathBuf, libfuzzer_target: PathBuf) { + assert_exists_and_is_file(&config).await; + assert_exists_and_is_file(&libfuzzer_target).await; + + let test_layout = create_test_directory(&config, &libfuzzer_target) + .await + .expect("Failed to create test directory layout"); + + info!("Executed test from: {:?}", &test_layout.root); + info!("Running template for 1 minute..."); + if let Ok(template_result) = timeout( + Duration::from_secs(60), + template::launch(&test_layout.config, None), + ) + .await + { + // Something went wrong when running the template so lets print out the template to be helpful + info!("Printing config as it was used in the test:"); + info!("{:?}", fs::read_to_string(&test_layout.config).await); + template_result.unwrap(); + } + + verify_test_layout_structure_did_not_change(&test_layout).await; + assert_directory_is_not_empty(&test_layout.inputs).await; + assert_directory_is_not_empty(&test_layout.crashes).await; + verify_coverage_dir(&test_layout.coverage).await; + + let _ = fs::remove_dir_all(&test_layout.root).await; +} + +async fn verify_test_layout_structure_did_not_change(test_layout: &TestLayout) { + assert_exists_and_is_dir(&test_layout.root).await; + assert_exists_and_is_file(&test_layout.config).await; + assert_exists_and_is_file(&test_layout.target_exe).await; + assert_exists_and_is_dir(&test_layout.crashdumps).await; + assert_exists_and_is_dir(&test_layout.coverage).await; + assert_exists_and_is_dir(&test_layout.crashes).await; + assert_exists_and_is_dir(&test_layout.inputs).await; + assert_exists_and_is_dir(&test_layout.regression_reports).await; +} + +async fn verify_coverage_dir(coverage: &Path) { + warn_if_empty(coverage).await; +} + +async fn assert_exists_and_is_dir(dir: &Path) { + assert!(dir.exists(), "Expected directory to exist. dir = {:?}", dir); + assert!( + dir.is_dir(), + "Expected path to be a directory. dir = {:?}", + dir + ); +} + +async fn warn_if_empty(dir: &Path) { + if dir_is_empty(dir).await { + println!("Expected directory to not be empty: {:?}", dir); + } +} + +async fn assert_exists_and_is_file(file: &Path) { + assert!(file.exists(), "Expected file to exist. file = {:?}", file); + assert!( + file.is_file(), + "Expected path to be a file. file = {:?}", + file + ); +} + +async fn dir_is_empty(dir: &Path) -> bool { + fs::read_dir(dir) + .await + .unwrap_or_else(|_| panic!("Failed to list files in directory. dir = {:?}", dir)) + .next_entry() + .await + .unwrap_or_else(|_| { + panic!( + "Failed to get next file in directory listing. dir = {:?}", + dir + ) + }) + .is_some() +} + +async fn assert_directory_is_not_empty(dir: &Path) { + assert!( + dir_is_empty(dir).await, + "Expected directory to not be empty. dir = {:?}", + dir + ); +} + +async fn create_test_directory(config: &Path, target_exe: &Path) -> Result { + let mut test_directory = PathBuf::from(".").join(uuid::Uuid::new_v4().to_string()); + fs::create_dir_all(&test_directory).await?; + test_directory = test_directory.canonicalize()?; + + let mut inputs_directory = PathBuf::from(&test_directory).join("inputs"); + fs::create_dir(&inputs_directory).await?; + inputs_directory = inputs_directory.canonicalize()?; + + let mut crashes_directory = PathBuf::from(&test_directory).join("crashes"); + fs::create_dir(&crashes_directory).await?; + crashes_directory = crashes_directory.canonicalize()?; + + let mut crashdumps_directory = PathBuf::from(&test_directory).join("crashdumps"); + fs::create_dir(&crashdumps_directory).await?; + crashdumps_directory = crashdumps_directory.canonicalize()?; + + let mut coverage_directory = PathBuf::from(&test_directory).join("coverage"); + fs::create_dir(&coverage_directory).await?; + coverage_directory = coverage_directory.canonicalize()?; + + let mut regression_reports_directory = + PathBuf::from(&test_directory).join("regression_reports"); + fs::create_dir(®ression_reports_directory).await?; + regression_reports_directory = regression_reports_directory.canonicalize()?; + + let mut target_in_test = PathBuf::from(&test_directory).join("fuzz.exe"); + fs::copy(target_exe, &target_in_test).await?; + target_in_test = target_in_test.canonicalize()?; + + let mut interesting_extensions = HashSet::new(); + interesting_extensions.insert(Some(OsStr::new("so"))); + interesting_extensions.insert(Some(OsStr::new("pdb"))); + let mut f = fs::read_dir(target_exe.parent().unwrap()).await?; + while let Ok(Some(f)) = f.next_entry().await { + if interesting_extensions.contains(&f.path().extension()) { + fs::copy(f.path(), PathBuf::from(&test_directory).join(f.file_name())).await?; + } + } + + let mut config_data = fs::read_to_string(config).await?; + + config_data = config_data + .replace("{TARGET_PATH}", target_in_test.to_str().unwrap()) + .replace("{INPUTS_PATH}", inputs_directory.to_str().unwrap()) + .replace("{CRASHES_PATH}", crashes_directory.to_str().unwrap()) + .replace("{CRASHDUMPS_PATH}", crashdumps_directory.to_str().unwrap()) + .replace("{COVERAGE_PATH}", coverage_directory.to_str().unwrap()) + .replace( + "{REGRESSION_REPORTS_PATH}", + regression_reports_directory.to_str().unwrap(), + ) + .replace("{TEST_DIRECTORY}", test_directory.to_str().unwrap()); + + let mut config_in_test = + PathBuf::from(&test_directory).join(config.file_name().unwrap_or_else(|| { + panic!("Failed to get file name for config. config = {:?}", config) + })); + + fs::write(&config_in_test, &config_data).await?; + config_in_test = config_in_test.canonicalize()?; + + Ok(TestLayout { + root: test_directory, + config: config_in_test, + target_exe: target_in_test, + inputs: inputs_directory, + crashes: crashes_directory, + crashdumps: crashdumps_directory, + coverage: coverage_directory, + regression_reports: regression_reports_directory, + }) +} + +#[derive(Debug)] +struct TestLayout { + root: PathBuf, + config: PathBuf, + target_exe: PathBuf, + inputs: PathBuf, + crashes: PathBuf, + crashdumps: PathBuf, + coverage: PathBuf, + regression_reports: PathBuf, +} diff --git a/src/agent/onefuzz-task/tests/templates/libfuzzer_basic.yml b/src/agent/onefuzz-task/tests/templates/libfuzzer_basic.yml new file mode 100644 index 0000000000..f6740cbc96 --- /dev/null +++ b/src/agent/onefuzz-task/tests/templates/libfuzzer_basic.yml @@ -0,0 +1,33 @@ +# yaml-language-server: $schema=../../src/local/schema.json + +required_args: &required_args + target_exe: '{TARGET_PATH}' + inputs: &inputs '{INPUTS_PATH}' # A folder containining your inputs + crashes: &crashes '{CRASHES_PATH}' # The folder where you want the crashing inputs to be output + crashdumps: '{CRASHDUMPS_PATH}' # The folder where you want the crash dumps to be output + coverage: '{COVERAGE_PATH}' # The folder where you want the code coverage to be output + regression_reports: '{REGRESSION_REPORTS_PATH}' # The folder where you want the regression reports to be output + target_env: { + 'LD_LIBRARY_PATH': '{TEST_DIRECTORY}', + } + target_options: [] + check_fuzzer_help: false + +tasks: + - type: LibFuzzer + <<: *required_args + readonly_inputs: [] + + - type: LibfuzzerRegression + <<: *required_args + + - type: "LibfuzzerCrashReport" + <<: *required_args + input_queue: *crashes + + - type: "Coverage" + <<: *required_args + target_options: + - "{input}" + input_queue: *inputs + readonly_inputs: [*inputs] diff --git a/src/agent/onefuzz-telemetry/Cargo.toml b/src/agent/onefuzz-telemetry/Cargo.toml index 23574a013f..8f91478b1f 100644 --- a/src/agent/onefuzz-telemetry/Cargo.toml +++ b/src/agent/onefuzz-telemetry/Cargo.toml @@ -15,5 +15,5 @@ chrono = { version = "0.4", default-features = false, features = [ lazy_static = "1.4" log = "0.4" serde = { version = "1.0", features = ["derive"] } -tokio = { version = "1.29", features = ["full"] } +tokio = { version = "1.32", features = ["full"] } uuid = { version = "1.4", features = ["serde", "v4"] } diff --git a/src/agent/onefuzz/Cargo.toml b/src/agent/onefuzz/Cargo.toml index 1f3c27985c..55042607fa 100644 --- a/src/agent/onefuzz/Cargo.toml +++ b/src/agent/onefuzz/Cargo.toml @@ -10,7 +10,7 @@ license = "MIT" anyhow = "1.0" async-trait = "0.1" base64 = "0.21" -bytes = "1.4" +bytes = "1.5" dunce = "1.0" dynamic-library = { path = "../dynamic-library" } futures = "0.3" @@ -18,7 +18,7 @@ futures-util = "0.3" hex = "0.4" lazy_static = "1.4" log = "0.4" -notify = { version = "6.0.1", default-features = false } +notify = { version = "6.1.1", default-features = false } regex = "1.9.1" reqwest = { version = "0.11", features = [ "json", @@ -31,7 +31,7 @@ serde = "1.0" serde_json = "1.0" rand = "0.8" serde_derive = "1.0" -tokio = { version = "1.29", features = ["full"] } +tokio = { version = "1.32", features = ["full"] } tokio-stream = { version = "0.1", features = ["fs", "time", "tokio-util"] } tokio-util = { version = "0.7", features = ["full"] } uuid = { version = "1.4", features = ["serde", "v4"] } @@ -40,7 +40,7 @@ url-escape = "0.1.0" storage-queue = { path = "../storage-queue" } strum = "0.25" strum_macros = "0.25" -tempfile = "3.7.0" +tempfile = "3.8.0" process_control = "4.0" reqwest-retry = { path = "../reqwest-retry" } onefuzz-telemetry = { path = "../onefuzz-telemetry" } @@ -49,7 +49,7 @@ stacktrace-parser = { path = "../stacktrace-parser" } backoff = { version = "0.4", features = ["tokio"] } [target.'cfg(target_family = "windows")'.dependencies] -winreg = "0.50" +winreg = "0.51" input-tester = { path = "../input-tester" } debugger = { path = "../debugger" } windows = { version = "0.48", features = [ @@ -62,10 +62,10 @@ cpp_demangle = "0.4" nix = "0.26" [target.'cfg(target_os = "linux")'.dependencies] -pete = "0.10" +pete = "0.12" rstack = "0.3" proc-maps = { version = "0.3", default-features = false } [dev-dependencies] -clap = { version = "4.3.0", features = ["derive"] } +clap = { version = "4.4.2", features = ["derive"] } pretty_assertions = "1.4.0" diff --git a/src/agent/onefuzz/src/libfuzzer.rs b/src/agent/onefuzz/src/libfuzzer.rs index 495f401bae..00b24bf4e9 100644 --- a/src/agent/onefuzz/src/libfuzzer.rs +++ b/src/agent/onefuzz/src/libfuzzer.rs @@ -339,7 +339,7 @@ impl LibFuzzer { Ok(missing) } - pub async fn fuzz( + pub fn fuzz( &self, fault_dir: impl AsRef, corpus_dir: impl AsRef, @@ -352,8 +352,7 @@ impl LibFuzzer { // specify that a new file `crash-` should be written to a // _directory_ ``, we must ensure that the prefix includes a // trailing path separator. - let artifact_prefix: OsString = - format!("-artifact_prefix={}/", fault_dir.as_ref().display()).into(); + let artifact_prefix = artifact_prefix(fault_dir.as_ref()); let mut cmd = self.build_command( Some(fault_dir.as_ref()), @@ -363,10 +362,11 @@ impl LibFuzzer { None, )?; + debug!("Running command: {:?}", &cmd); + let child = cmd .spawn() .with_context(|| format_err!("libfuzzer failed to start: {}", self.exe.display()))?; - Ok(child) } @@ -441,6 +441,20 @@ impl LibFuzzer { } } +#[cfg(target_os = "windows")] +fn artifact_prefix(fault_dir: &Path) -> OsString { + if fault_dir.is_absolute() { + format!("-artifact_prefix={}\\", fault_dir.display()).into() + } else { + format!("-artifact_prefix={}/", fault_dir.display()).into() + } +} + +#[cfg(not(target_os = "windows"))] +fn artifact_prefix(fault_dir: &Path) -> OsString { + format!("-artifact_prefix={}/", fault_dir.display()).into() +} + pub struct LibFuzzerLine { _line: String, iters: u64, diff --git a/src/agent/reqwest-retry/Cargo.toml b/src/agent/reqwest-retry/Cargo.toml index d7d12ff4e8..5ddfbbe419 100644 --- a/src/agent/reqwest-retry/Cargo.toml +++ b/src/agent/reqwest-retry/Cargo.toml @@ -19,5 +19,5 @@ reqwest = { version = "0.11", features = [ thiserror = "1.0" [dev-dependencies] -tokio = { version = "1.29", features = ["macros"] } +tokio = { version = "1.32", features = ["macros"] } wiremock = "0.5" diff --git a/src/agent/storage-queue/Cargo.toml b/src/agent/storage-queue/Cargo.toml index d5c1c09d08..58034cff18 100644 --- a/src/agent/storage-queue/Cargo.toml +++ b/src/agent/storage-queue/Cargo.toml @@ -10,7 +10,7 @@ anyhow = "1.0" async-trait = "0.1" backoff = { version = "0.4", features = ["tokio"] } base64 = "0.21" -bytes = { version = "1.4", features = ["serde"] } +bytes = { version = "1.5", features = ["serde"] } derivative = "2.2" flume = "0.10" num_cpus = "1.15" @@ -26,6 +26,6 @@ serde = { version = "1.0", features = ["derive"] } serde_derive = "1.0" serde_json = "1.0" bincode = "1.3" -tokio = { version = "1.29", features = ["full"] } +tokio = { version = "1.32", features = ["full"] } queue-file = "1.4" uuid = { version = "1.4", features = ["serde", "v4"] } diff --git a/src/agent/win-util/Cargo.toml b/src/agent/win-util/Cargo.toml index 1edaa3fc58..460ee0e197 100644 --- a/src/agent/win-util/Cargo.toml +++ b/src/agent/win-util/Cargo.toml @@ -12,7 +12,7 @@ log = "0.4" os_pipe = "1.1" [target.'cfg(windows)'.dependencies] -winreg = "0.50" +winreg = "0.51" [dependencies.windows] version = "0.48" @@ -33,4 +33,4 @@ features = [ ] [dev-dependencies] -tempfile = "3.7.0" +tempfile = "3.8.0" diff --git a/src/ci/agent.sh b/src/ci/agent.sh index 4a49c975b3..4cca93168b 100755 --- a/src/ci/agent.sh +++ b/src/ci/agent.sh @@ -37,7 +37,7 @@ export RUST_BACKTRACE=full # Run tests and collect coverage # https://github.com/taiki-e/cargo-llvm-cov -cargo llvm-cov nextest --all-targets --features slow-tests --locked --workspace --lcov --output-path "$output_dir/lcov.info" +cargo llvm-cov nextest --all-targets --features slow-tests,integration_test --locked --workspace --lcov --output-path "$output_dir/lcov.info" # TODO: re-enable integration tests. # cargo test --release --manifest-path ./onefuzz-task/Cargo.toml --features integration_test -- --nocapture diff --git a/src/ci/set-versions.sh b/src/ci/set-versions.sh index 34c30ea37c..2271a752f4 100755 --- a/src/ci/set-versions.sh +++ b/src/ci/set-versions.sh @@ -10,8 +10,12 @@ GET_VERSION=${SCRIPT_DIR}/get-version.sh VERSION=${1:-$(${GET_VERSION})} cd ${SCRIPT_DIR}/../../ +arrVer=(${VERSION//./ }) +MAJOR=${arrVer[0]} +MINOR=${arrVer[1]} + SET_VERSIONS="src/pytypes/onefuzztypes/__version__.py src/cli/onefuzz/__version__.py" SET_REQS="src/cli/requirements.txt" sed -i "s/0.0.0/${VERSION}/" ${SET_VERSIONS} -sed -i "s/onefuzztypes==0.0.0/onefuzztypes==${VERSION}/" ${SET_REQS} +sed -i "s/onefuzztypes==0.0.0/onefuzztypes==${MAJOR}.${MINOR}.*/" ${SET_REQS} diff --git a/src/proxy-manager/Cargo.lock b/src/proxy-manager/Cargo.lock index ca4813995e..e86da294a7 100644 --- a/src/proxy-manager/Cargo.lock +++ b/src/proxy-manager/Cargo.lock @@ -43,16 +43,15 @@ dependencies = [ [[package]] name = "anstream" -version = "0.3.2" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" +checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c" dependencies = [ "anstyle", "anstyle-parse", "anstyle-query", "anstyle-wincon", "colorchoice", - "is-terminal", "utf8parse", ] @@ -82,9 +81,9 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "1.0.2" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c677ab05e09154296dd37acecd46420c17b9713e8366facafa8fc0885167cf4c" +checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd" dependencies = [ "anstyle", "windows-sys", @@ -236,23 +235,22 @@ dependencies = [ [[package]] name = "clap" -version = "4.3.21" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c27cdf28c0f604ba3f512b0c9a409f8de8513e4816705deb0498b627e7c3a3fd" +checksum = "6a13b88d2c62ff462f88e4a121f17a82c1af05693a2f192b5c38d14de73c19f6" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" -version = "4.3.21" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08a9f1ab5e9f01a9b81f202e8562eb9a10de70abf9eaeac1be465c28b75aa4aa" +checksum = "2bb9faaa7c2ef94b2743a21f5a29e6f0010dff4caa69ac8e9d6cf8b6fa74da08" dependencies = [ "anstream", "anstyle", "clap_lex", - "once_cell", "strsim", ] @@ -1474,9 +1472,9 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.7.1" +version = "3.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc02fddf48964c42031a0b3fe0428320ecf3a73c401040fc0096f97794310651" +checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" dependencies = [ "cfg-if", "fastrand", @@ -1531,9 +1529,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.30.0" +version = "1.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d3ce25f50619af8b0aec2eb23deebe84249e19e2ddd393a6e16e3300a6dadfd" +checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" dependencies = [ "backtrace", "bytes", diff --git a/src/proxy-manager/Cargo.toml b/src/proxy-manager/Cargo.toml index c783e8d3aa..b2258e994b 100644 --- a/src/proxy-manager/Cargo.toml +++ b/src/proxy-manager/Cargo.toml @@ -20,7 +20,7 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" storage-queue = { path = "../agent/storage-queue" } thiserror = "1.0" -tokio = { version = "1.29", features = [ +tokio = { version = "1.32", features = [ "macros", "rt-multi-thread", "fs", @@ -31,4 +31,4 @@ reqwest-retry = { path = "../agent/reqwest-retry" } onefuzz-telemetry = { path = "../agent/onefuzz-telemetry" } uuid = { version = "1.4", features = ["serde"] } log = "0.4" -tempfile = "3.7.0" +tempfile = "3.8.0" diff --git a/src/pytypes/onefuzztypes/models.py b/src/pytypes/onefuzztypes/models.py index a5f8139e97..c888621600 100644 --- a/src/pytypes/onefuzztypes/models.py +++ b/src/pytypes/onefuzztypes/models.py @@ -273,6 +273,7 @@ class ADOTemplate(BaseModel): unique_fields: List[str] comment: Optional[str] ado_fields: Dict[str, str] + ado_duplicate_fields: Optional[Dict[str, str]] on_duplicate: ADODuplicateTemplate # validator needed to convert auth_token to SecretData