Merge pull request #797 from schungx/master

Fix fuzz bugs.

CHANGELOG.md

@@ -14,6 +14,7 @@ Potentially breaking changes
 * `ImmutableString` now derefs to `&str` instead of `&SmartString`. Normally this should not be a breaking change.
 * Traits implemented by `ImmutableString` are cleaned up. Normally this should not be a breaking change.
 * `EvalContext::new`, `FloatWrapper` and `ConditionalExpr` are now gated under `internals`.
+* Previously, Rhai follows [Unicode's definition for _whitespace_](https://en.wikipedia.org/wiki/Template:Whitespace_(Unicode)), which allows many exotic whitespace characters in scripts. Starting from this version, whitespace is strictly defined as the set of six ASCII characters (TAB, SPACE, CR, LF, V-TAB and FF). All other Unicode whitespace characters (not inside strings) are not considered whitespace by Rhai. If a script used to contain non-ASCII whitespace characters, it now fails to parse with a syntax error.
 
 Deprecated API's
 ----------------

fuzz/fuzz_targets/scripting.rs

@@ -24,13 +24,13 @@ fuzz_target!(|ctx: Ctx| {
     engine.set_optimization_level(ctx.optimization_level);
 
     // Limit the length of scripts.
-    let script = &ctx.script[..(ctx.script.len().min(32 * 1020))];
+    let script = ctx.script.chars().take(32 * 1024).collect::<String>();
 
     // We need fuzzing to be fast, so we'll stop executing after 1s.
     let start = Instant::now();
     engine.on_progress(move |_| (start.elapsed().as_millis() > 1000).then_some(Dynamic::UNIT));
 
     let engine = engine;
 
-    _ = engine.run(script);
+    _ = engine.run(&script);
 });

src/tokenizer.rs

@@ -2218,8 +2218,8 @@ fn get_next_token_inner(
             // \n
             ('\n', ..) => pos.new_line(),
 
-            // Whitespace
-            (' ' | '\t' | '\r', ..) => (),
+            // Whitespace - follows JavaScript's SPACE, TAB, CR, V-TAB, FF
+            (' ' | '\t' | '\r' | '\x0b' | '\x0c', ..) => (),
 
             _ => {
                 return (