From ffdfe3ec80009dd2784ad91dc97a5c4c2289f2fa Mon Sep 17 00:00:00 2001 From: Kisaragi Marine Date: Sun, 22 Oct 2023 18:46:28 +0900 Subject: [PATCH 1/3] test: enable position test of underscore_discard --- Cargo.lock | 1 + package/origlang-testsuite/Cargo.toml | 1 + package/origlang-testsuite/src/main.rs | 22 +++++++++++++--------- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a22dc699..938d112e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1078,6 +1078,7 @@ dependencies = [ "origlang-ir", "origlang-ir-optimizer", "origlang-runtime", + "origlang-source-span", "thiserror", ] diff --git a/package/origlang-testsuite/Cargo.toml b/package/origlang-testsuite/Cargo.toml index cce8900e..cc5d427e 100644 --- a/package/origlang-testsuite/Cargo.toml +++ b/package/origlang-testsuite/Cargo.toml @@ -13,4 +13,5 @@ origlang-compiler = { version = "0.1.0", path = "../origlang-compiler" } origlang-ir = { version = "0.1.0", path = "../origlang-ir" } origlang-ir-optimizer = { version = "0.1.0", path = "../origlang-ir-optimizer" } origlang-runtime = { version = "0.1.0", path = "../origlang-runtime" } +origlang-source-span = { version = "*", path = "../origlang-source-span" } thiserror = "1.0.50" diff --git a/package/origlang-testsuite/src/main.rs b/package/origlang-testsuite/src/main.rs index a2d879cb..d422e3f1 100644 --- a/package/origlang-testsuite/src/main.rs +++ b/package/origlang-testsuite/src/main.rs @@ -19,10 +19,11 @@ use origlang_compiler::type_check::TypeChecker; use origlang_ir::IntoVerbatimSequencedIR; use origlang_ir_optimizer::lower::{EachStep, LowerStep, TheTranspiler}; use origlang_ir_optimizer::preset::{NoOptimization, SimpleOptimization}; +use origlang_source_span::SourcePosition; type Err = TestFailureCause; -#[derive(Error, Debug)] +#[derive(Error, Debug, Eq, PartialEq)] pub enum TestFailureCause { #[error("parser failure: {0}")] Parser(#[from] SimpleErrorWithPos), @@ -461,17 +462,20 @@ print 1 assert_eq!(Self::evaluated_expressions("var _ = 1\n")?, []); assert_eq!(Self::evaluated_expressions("var a = block\n print 1\n()\nend\n").expect("FATAL: shouldn't fail"), type_boxes![ 1 => NonCoercedInteger ]); assert_eq!(Self::evaluated_expressions("var _ = block\n print 1\n()\nend\n")?, type_boxes![ 1 => NonCoercedInteger ]); - assert!( - matches!(Self::evaluated_expressions("var _ = _\n"), - Err( - TestFailureCause::Parser( - SimpleErrorWithPos { - kind: ParserError::UnderscoreCanNotBeRightHandExpression, - .. + assert_eq!( + Self::evaluated_expressions("var _ = _\n"), + Err( + TestFailureCause::Parser( + SimpleErrorWithPos { + kind: ParserError::UnderscoreCanNotBeRightHandExpression, + position: SourcePosition { + line: 1.try_into().unwrap(), + column: 9.try_into().unwrap(), } - ) + } ) ) + ); Ok(()) From 3f4664b33b7c619fd6fddd5482c85341595e32ee Mon Sep 17 00:00:00 2001 From: Kisaragi Marine Date: Sun, 22 Oct 2023 19:53:19 +0900 Subject: [PATCH 2/3] refactor: wrap LC-operation in integrated struct --- package/origlang-compiler/src/lexer.rs | 58 +++++++++++++++++++------- 1 file changed, 43 insertions(+), 15 deletions(-) diff --git a/package/origlang-compiler/src/lexer.rs b/package/origlang-compiler/src/lexer.rs index 1b2ba9a7..87adf423 100644 --- a/package/origlang-compiler/src/lexer.rs +++ b/package/origlang-compiler/src/lexer.rs @@ -32,12 +32,19 @@ impl AssociateWithPos for T { } } +#[derive(Debug)] +pub struct LcManager { + /// don't use directly. + line: Cell, + /// don't use directly. + column: Cell, +} + #[derive(Debug)] pub struct Lexer<'src> { source_bytes_nth: Cell, source: &'src str, - line: Cell, - column: Cell, + lc_manager: LcManager, } impl<'src> Lexer<'src> { @@ -50,10 +57,12 @@ impl<'src> Lexer<'src> { Self { source_bytes_nth: Cell::new(Utf8CharBoundaryStartByte::new(0)), source, - // SAFETY: 1 != 0 - line: Cell::new(unsafe { NonZeroUsize::new_unchecked(1) }), - // SAFETY: 1 != 0 - column: Cell::new(unsafe { NonZeroUsize::new_unchecked(1) }), + lc_manager: LcManager { + // SAFETY: 1 != 0 + line: Cell::new(unsafe { NonZeroUsize::new_unchecked(1) }), + // SAFETY: 1 != 0 + column: Cell::new(unsafe { NonZeroUsize::new_unchecked(1) }), + } } } } @@ -254,6 +263,7 @@ impl Lexer<'_> { } pub fn next(&self) -> WithPosition { + debug!("-------------------------------------------------"); self.drain_space(); if self.reached_end() { @@ -274,8 +284,8 @@ impl Lexer<'_> { fn current_pos(&self) -> SourcePos { SourcePos { - line: self.line.get(), - column: self.column.get(), + line: self.line(), + column: self.column(), } } @@ -352,19 +362,20 @@ impl Lexer<'_> { fn set_current_index(&self, future_index: Utf8CharBoundaryStartByte) -> Result<(), OutOfRangeError> { let old = self.source_bytes_nth.get().as_usize(); let new = future_index.as_usize(); + debug!("index: {old} -> {future_index:?}"); if old == new { return Ok(()) } - let current_line = self.line.get().get(); + let current_line = self.line().get(); let src = &self.source; if old < new { // forward let new_line = current_line + src[old..new].bytes().filter(|x| *x == b'\n').count(); let new_col = src[old..new].rfind('\n').map_or_else(|| { - let mut c = self.column.get().get(); + let mut c = self.column().get(); c += new - old; c @@ -372,8 +383,8 @@ impl Lexer<'_> { new - (old + old_relative) }); - self.line.set(NonZeroUsize::new(new_line).expect("overflow")); - self.column.set(NonZeroUsize::new(new_col).expect("overflow")); + self.set_line(NonZeroUsize::new(new_line).expect("overflow")); + self.set_column(NonZeroUsize::new(new_col).expect("overflow")); } else { // back let new_line = current_line - src[new..old].bytes().filter(|x| *x == b'\n').count(); @@ -398,11 +409,10 @@ impl Lexer<'_> { }) }); - self.line.set(NonZeroUsize::new(new_line).expect("overflow")); - self.column.set(NonZeroUsize::new(new_col).expect("overflow")); + self.set_line(NonZeroUsize::new(new_line).expect("overflow")); + self.set_column(NonZeroUsize::new(new_col).expect("overflow")); } - debug!("index: requested = {future_index:?}"); self.source_bytes_nth.set(future_index); Ok(()) @@ -548,4 +558,22 @@ impl Lexer<'_> { max: self.source.len(), }) } + + fn line(&self) -> NonZeroUsize { + self.lc_manager.line.get() + } + + fn set_line(&self, line: NonZeroUsize) { + debug!("line: {old} -> {new}", old = self.line(), new = line); + self.lc_manager.line.set(line) + } + + fn column(&self) -> NonZeroUsize { + self.lc_manager.column.get() + } + + fn set_column(&self, column: NonZeroUsize) { + debug!("column: {old} -> {new}", old = self.column(), new = column); + self.lc_manager.column.set(column) + } } From cc336a81d7955b52ff0ec21455ad867f3347df1f Mon Sep 17 00:00:00 2001 From: Kisaragi Marine Date: Sun, 22 Oct 2023 19:54:35 +0900 Subject: [PATCH 3/3] fix: another LC-bug, it does not handle backward case correctly --- package/origlang-compiler/src/lexer.rs | 5 +++-- package/origlang-compiler/src/lexer/token.rs | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/package/origlang-compiler/src/lexer.rs b/package/origlang-compiler/src/lexer.rs index 87adf423..4041a00b 100644 --- a/package/origlang-compiler/src/lexer.rs +++ b/package/origlang-compiler/src/lexer.rs @@ -386,11 +386,12 @@ impl Lexer<'_> { self.set_line(NonZeroUsize::new(new_line).expect("overflow")); self.set_column(NonZeroUsize::new(new_col).expect("overflow")); } else { + // THIS BRANCH IS IMPORTANT!!! OTHERWISE, RESET OPERATION WILL NOT WORK!!! // back let new_line = current_line - src[new..old].bytes().filter(|x| *x == b'\n').count(); let new_col = src[new..old].find('\n').map_or_else(|| { - let mut c = self.column.get().get(); - c += old - new; + let mut c = self.column().get(); + c -= old - new; c }, |new_relative| { diff --git a/package/origlang-compiler/src/lexer/token.rs b/package/origlang-compiler/src/lexer/token.rs index a85c0bc5..f851557b 100644 --- a/package/origlang-compiler/src/lexer/token.rs +++ b/package/origlang-compiler/src/lexer/token.rs @@ -1,3 +1,4 @@ +use std::num::NonZeroUsize; use origlang_ast::{Comment, Identifier}; use crate::chars::boundary::Utf8CharBoundaryStartByte; use crate::lexer::Lexer; @@ -16,7 +17,7 @@ impl TemporalLexerUnwindToken { } pub fn reset(self, lexer: &Lexer) { - lexer.source_bytes_nth.set(self.unwind_index); + lexer.set_current_index(self.unwind_index).expect("Error during reset"); } }