Fine tune tokenizer.

schungx · Aug 30, 2023 · 1fb46f8 · 1fb46f8
1 parent 5915ea5
commit 1fb46f8
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 19 deletions.
diff --git a/src/defer.rs b/src/defer.rs
@@ -41,11 +41,9 @@ macro_rules! defer {
     };
     ($var:ident = ( $value:expr ) if Some($guard:ident) => $restore:expr) => {
         let mut __rx__;
-        let $var = if let Some($guard) = $guard {
-            __rx__ = crate::Deferred::lock($value, $restore);
-            &mut *__rx__
-        } else {
-            &mut *$value
+        let $var = match $guard {
+            Some($guard) => { __rx__ = crate::Deferred::lock($value, $restore); &mut *__rx__ }
+            _ => &mut *$value
         };
     };
     ($var:ident if $guard:expr => $restore:expr) => {

diff --git a/src/func/builtin.rs b/src/func/builtin.rs
@@ -433,12 +433,12 @@ pub fn get_builtin_binary_op_fn(op: &Token, x: &Dynamic, y: &Dynamic) -> Option<
 
     // char op string
     if (type1, type2) == (TypeId::of::<char>(), TypeId::of::<ImmutableString>()) {
-        fn get_s1s2(args: &FnCallArgs) -> ([char; 2], [char; 2]) {
+        fn get_s1s2(args: &FnCallArgs) -> ([Option<char>; 2], [Option<char>; 2]) {
             let x = args[0].as_char().unwrap();
             let y = &*args[1].read_lock::<ImmutableString>().unwrap();
-            let s1 = [x, '\0'];
+            let s1 = [Some(x), None];
             let mut y = y.chars();
-            let s2 = [y.next().unwrap_or('\0'), y.next().unwrap_or('\0')];
+            let s2 = [y.next(), y.next()];
             (s1, s2)
         }
 
@@ -470,12 +470,12 @@ pub fn get_builtin_binary_op_fn(op: &Token, x: &Dynamic, y: &Dynamic) -> Option<
     }
     // string op char
     if (type1, type2) == (TypeId::of::<ImmutableString>(), TypeId::of::<char>()) {
-        fn get_s1s2(args: &FnCallArgs) -> ([char; 2], [char; 2]) {
+        fn get_s1s2(args: &FnCallArgs) -> ([Option<char>; 2], [Option<char>; 2]) {
             let x = &*args[0].read_lock::<ImmutableString>().unwrap();
             let y = args[1].as_char().unwrap();
             let mut x = x.chars();
-            let s1 = [x.next().unwrap_or('\0'), x.next().unwrap_or('\0')];
-            let s2 = [y, '\0'];
+            let s1 = [x.next(), x.next()];
+            let s2 = [Some(y), None];
             (s1, s2)
         }
 

diff --git a/src/tokenizer.rs b/src/tokenizer.rs
@@ -1608,24 +1608,24 @@ fn get_next_token_inner(
                             stream.get_next().unwrap();
 
                             // Check if followed by digits or something that cannot start a property name
-                            match stream.peek_next().unwrap_or('\0') {
+                            match stream.peek_next() {
                                 // digits after period - accept the period
-                                '0'..='9' => {
+                                Some('0'..='9') => {
                                     result.push(next_char);
                                     pos.advance();
                                 }
                                 // _ - cannot follow a decimal point
-                                NUMBER_SEPARATOR => {
+                                Some(NUMBER_SEPARATOR) => {
                                     stream.unget(next_char);
                                     break;
                                 }
                                 // .. - reserved symbol, not a floating-point number
-                                '.' => {
+                                Some('.') => {
                                     stream.unget(next_char);
                                     break;
                                 }
                                 // symbol after period - probably a float
-                                ch if !is_id_first_alphabetic(ch) => {
+                                Some(ch) if !is_id_first_alphabetic(ch) => {
                                     result.push(next_char);
                                     pos.advance();
                                     result.push('0');
@@ -1642,14 +1642,14 @@ fn get_next_token_inner(
                             stream.get_next().expect("`e`");
 
                             // Check if followed by digits or +/-
-                            match stream.peek_next().unwrap_or('\0') {
+                            match stream.peek_next() {
                                 // digits after e - accept the e
-                                '0'..='9' => {
+                                Some('0'..='9') => {
                                     result.push(next_char);
                                     pos.advance();
                                 }
                                 // +/- after e - accept the e and the sign
-                                '+' | '-' => {
+                                Some('+' | '-') => {
                                     result.push(next_char);
                                     pos.advance();
                                     result.push(stream.get_next().unwrap());