From 1fb46f898f95648f33c105fa5bf7d4f9a964b7c5 Mon Sep 17 00:00:00 2001
From: Stephen Chung <schungx@live.com>
Date: Wed, 30 Aug 2023 22:51:32 +0800
Subject: [PATCH] Fine tune tokenizer.

---
 src/defer.rs        |  8 +++-----
 src/func/builtin.rs | 12 ++++++------
 src/tokenizer.rs    | 16 ++++++++--------
 3 files changed, 17 insertions(+), 19 deletions(-)

diff --git a/src/defer.rs b/src/defer.rs
index 4bf0ff21f..aed0176a6 100644
--- a/src/defer.rs
+++ b/src/defer.rs
@@ -41,11 +41,9 @@ macro_rules! defer {
     };
     ($var:ident = ( $value:expr ) if Some($guard:ident) => $restore:expr) => {
         let mut __rx__;
-        let $var = if let Some($guard) = $guard {
-            __rx__ = crate::Deferred::lock($value, $restore);
-            &mut *__rx__
-        } else {
-            &mut *$value
+        let $var = match $guard {
+            Some($guard) => { __rx__ = crate::Deferred::lock($value, $restore); &mut *__rx__ }
+            _ => &mut *$value
         };
     };
     ($var:ident if $guard:expr => $restore:expr) => {
diff --git a/src/func/builtin.rs b/src/func/builtin.rs
index 629fc80fc..c191ab7ef 100644
--- a/src/func/builtin.rs
+++ b/src/func/builtin.rs
@@ -433,12 +433,12 @@ pub fn get_builtin_binary_op_fn(op: &Token, x: &Dynamic, y: &Dynamic) -> Option<
 
     // char op string
     if (type1, type2) == (TypeId::of::<char>(), TypeId::of::<ImmutableString>()) {
-        fn get_s1s2(args: &FnCallArgs) -> ([char; 2], [char; 2]) {
+        fn get_s1s2(args: &FnCallArgs) -> ([Option<char>; 2], [Option<char>; 2]) {
             let x = args[0].as_char().unwrap();
             let y = &*args[1].read_lock::<ImmutableString>().unwrap();
-            let s1 = [x, '\0'];
+            let s1 = [Some(x), None];
             let mut y = y.chars();
-            let s2 = [y.next().unwrap_or('\0'), y.next().unwrap_or('\0')];
+            let s2 = [y.next(), y.next()];
             (s1, s2)
         }
 
@@ -470,12 +470,12 @@ pub fn get_builtin_binary_op_fn(op: &Token, x: &Dynamic, y: &Dynamic) -> Option<
     }
     // string op char
     if (type1, type2) == (TypeId::of::<ImmutableString>(), TypeId::of::<char>()) {
-        fn get_s1s2(args: &FnCallArgs) -> ([char; 2], [char; 2]) {
+        fn get_s1s2(args: &FnCallArgs) -> ([Option<char>; 2], [Option<char>; 2]) {
             let x = &*args[0].read_lock::<ImmutableString>().unwrap();
             let y = args[1].as_char().unwrap();
             let mut x = x.chars();
-            let s1 = [x.next().unwrap_or('\0'), x.next().unwrap_or('\0')];
-            let s2 = [y, '\0'];
+            let s1 = [x.next(), x.next()];
+            let s2 = [Some(y), None];
             (s1, s2)
         }
 
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index 1113f14af..5fbdfae68 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -1608,24 +1608,24 @@ fn get_next_token_inner(
                             stream.get_next().unwrap();
 
                             // Check if followed by digits or something that cannot start a property name
-                            match stream.peek_next().unwrap_or('\0') {
+                            match stream.peek_next() {
                                 // digits after period - accept the period
-                                '0'..='9' => {
+                                Some('0'..='9') => {
                                     result.push(next_char);
                                     pos.advance();
                                 }
                                 // _ - cannot follow a decimal point
-                                NUMBER_SEPARATOR => {
+                                Some(NUMBER_SEPARATOR) => {
                                     stream.unget(next_char);
                                     break;
                                 }
                                 // .. - reserved symbol, not a floating-point number
-                                '.' => {
+                                Some('.') => {
                                     stream.unget(next_char);
                                     break;
                                 }
                                 // symbol after period - probably a float
-                                ch if !is_id_first_alphabetic(ch) => {
+                                Some(ch) if !is_id_first_alphabetic(ch) => {
                                     result.push(next_char);
                                     pos.advance();
                                     result.push('0');
@@ -1642,14 +1642,14 @@ fn get_next_token_inner(
                             stream.get_next().expect("`e`");
 
                             // Check if followed by digits or +/-
-                            match stream.peek_next().unwrap_or('\0') {
+                            match stream.peek_next() {
                                 // digits after e - accept the e
-                                '0'..='9' => {
+                                Some('0'..='9') => {
                                     result.push(next_char);
                                     pos.advance();
                                 }
                                 // +/- after e - accept the e and the sign
-                                '+' | '-' => {
+                                Some('+' | '-') => {
                                     result.push(next_char);
                                     pos.advance();
                                     result.push(stream.get_next().unwrap());