Bugfixes and updates, ready for ea+12

CHANGELOG.md

@@ -1,5 +1,30 @@
 # Changelog
 
+### 3.0 Early Access Preview 12 (21 December 2021)
+
+Implemented:
+- Support for various additional built-in functions, such as, e.g. `playTone`;
+- Runs existing TigerJython programs based on `gturtle`, etc.;
+- Updated version of Jython core (external GitHub repo) - includes support for `color`
+  as a 'native' data type in TigerJython;
+
+Bug fixes:
+- Syntax highlighting was erroneous, in part due to unexpected behaviour from the
+  `RichTextFX` component used for the editor;
+
+Known issues:
+- Syntax highlighting colour schemes are tentative and only available in selected themes;
+- Does not work with ARM architectures because of a bug in `sbt`/`maven` that prevents us
+  from updating JavaFX to the required version 17;
+- Java emits a warning because we package JavaFX with out application (rather than 
+  providing external modules): 
+  `WARNING: Unsupported JavaFX configuration: classes were loaded from`.  This can be
+  ignored;
+
+### 3.0 Early Access Previews 9-11
+
+*A series of minor releases for internal testing with various bug fixes.*
+
 ### 3.0 Early Access Preview 8 (2 August 2021)
 
 Implemented:

README.md

@@ -7,7 +7,9 @@ the [JAR-file](https://github.com/Tiger-Jython/TigerJython/releases/download/v3.
 
 > This is an early access version that is not yet ready for classroom use!!!
 
-**This version of TigerJython requires JRE version 9+ and will not run with JRE 8.**
+**This version of TigerJython requires JRE version 11+ and will not run with JRE 8.**
+One of the main reasons is that the JavaFX libraries used here requires an up-to-date
+Java version.
 
 
 ## Features and Design
@@ -63,6 +65,19 @@ TigerJython uses [`sbt`](https://www.scala-sbt.org/) to build the code.  Start `
 `compile` to compile the code and `assembly` to generate a JAR file containing all the necessary 
 libraries (such as Scala, Jython, etc).
 
+Note that this project depends on other sub-projects, which are included using symbolic links.  If you
+are running Windows, you might have to manually replace the links in the `resources` directory with the
+text files, etc. used for display.  There are three major sub-projects:
+- [TigerPython-Parser](https://github.com/Tobias-Kohn/TigerPython-Parser/)
+- [TigerJython-Localisation](https://github.com/Tiger-Jython/TigerJython-Localisation)
+- [TigerJython:Jython](https://github.com/Tiger-Jython/jython)
+
+Additionally, TigerJython also includes various libraries such as extended turtle graphics, etc.  You
+can find [some of these libraries on GitHub](https://github.com/Tiger-Jython/Aplu-Libraries), 
+but not all of them are open-sourced and generally available.  Please contact the authors if you require
+a copy of the full TigerJython support libraries (note: the editor and Jython will run perfectly fine 
+without these, but it will not provide the full functionality of TigerJython).
+
 By placing additional JARs into the `lib` subfolder, you can have additional files integrated into the
 project.
 

build.sbt

@@ -70,7 +70,8 @@ osName := (System.getProperty("os.name") match {
 })
 
 // We need the newest version as it contains important fixes for Mac OS X
-val fxVersion = "14.0.1" //  "11-ea+25"
+// Actually, version 17.0.1 is out but due to a bug in JavaFX maven the current version of SBT cannot pull it :-(
+val fxVersion = "16" //  "11-ea+25"
 
 /*libraryDependencies += "org.openjfx" % "javafx-base" % fxVersion classifier osName.value
 libraryDependencies += "org.openjfx" % "javafx-controls" % fxVersion classifier osName.value
@@ -94,6 +95,13 @@ libraryDependencies += "org.openjfx" % "javafx-controls" % fxVersion classifier
 libraryDependencies += "org.openjfx" % "javafx-fxml" % fxVersion classifier "mac"
 libraryDependencies += "org.openjfx" % "javafx-graphics" % fxVersion classifier "mac"
 
+// In order to include OpenJFX for ARM architecture, we need at least version 17.  However, due to a bug it is currently
+// not possible.  We have to wait for updates to SBT/Maven before being able to pull the correct versions.
+/* libraryDependencies += "org.openjfx" % "javafx-base" % fxVersion classifier "linux-aarch64"
+libraryDependencies += "org.openjfx" % "javafx-controls" % fxVersion classifier "linux-aarch64"
+libraryDependencies += "org.openjfx" % "javafx-fxml" % fxVersion classifier "linux-aarch64"
+libraryDependencies += "org.openjfx" % "javafx-graphics" % fxVersion classifier "linux-aarch64" */
+
 // Other dependencies
 libraryDependencies += "org.fxmisc.richtext" % "richtextfx" % "0.10.5"
 

src/main/scala/tigerjython/syntaxsupport/SyntaxDocument.scala

@@ -67,27 +67,27 @@ class SyntaxDocument {
       val (pos, index) = tokenIndexFromPosition(position)
       // Check for the special case where we are deleting whitespace, which requires no re-parsing
       tokens(index) match {
-        case whitespaceToken: WhitespaceToken if position + delLength <= pos + whitespaceToken.length =>
+        case whitespaceToken: WhitespaceToken if position + delLength <= pos + whitespaceToken.length && delLength < whitespaceToken.length =>
           whitespaceToken.length -= delLength
           text.delete(position, position + delLength)
           return
         case Token(TokenType.NEWLINE, len) if position == pos + len && index + 1 < tokens.length =>
           tokens(index + 1) match {
             case whitespaceToken: WhitespaceToken =>
-              if (whitespaceToken.length == delLength) {
+              /*if (whitespaceToken.length == delLength) {
                 tokens.remove(index + 1)
                 text.delete(position, position + delLength)
                 return
               }
-              else if (whitespaceToken.length > delLength) {
+              else*/ if (whitespaceToken.length > delLength) {
                 tokens(index + 1).length -= delLength
                 text.delete(position, position + delLength)
                 return
               }
             case _ =>
           }
         // Do not reparse names if we merely remove a single character
-        case token @ NameToken(TokenType.NAME, nameTokenType, tokenText) if pos < position &&
+        case token @ NameToken(TokenType.NAME, _, tokenText) if pos < position &&
             position + delLength < pos + tokenText.length =>
           val p = position - pos
           val s = new StringBuilder(tokenText).delete(p, p + delLength).toString()
@@ -165,6 +165,12 @@ class SyntaxDocument {
     } else
       -1
 
+  /**
+   * The `RichTextFX` component we use inserts control characters into the text before acting upon them.  Hence, when
+   * pressing the `delete` key, it first inserts `0x7F` and then deletes two characters.  These control characters
+   * upset the `SyntaxDocument` here.  Rather than ignoring these control characters while parsing the text, we simply
+   * throw away the messages of inserting them into our text in the first place.
+   */
   @inline
   private def _ignoreChar(c: Char): Boolean =
     if (c < ' ')
@@ -221,6 +227,10 @@ class SyntaxDocument {
               }
             case _ =>
           }
+        // Check if we are inserting a new line at the end of an existing line
+        if (insText == "\n" && tokens(index).tokenType == TokenType.NEWLINE) {
+          // ToDo
+        }
 
         val length =
           if (position == pos + tokens(index).length && index + 1 < tokens.length)
@@ -268,8 +278,10 @@ class SyntaxDocument {
           tokenizer.extendParseRange(l)
         }
       }
-      for (message <- tokens.createMessages())
+      for (message <- tokens.createMessages()) {
         struct.handleMessage(message.index, message)
+      }
+      StructLine.validateAstNodes()
     }
   }
 

src/main/scala/tigerjython/syntaxsupport/parser/PythonStmtParser.scala

@@ -16,7 +16,7 @@ import tigerjython.syntaxsupport.tokens._
  */
 class PythonStmtParser(val document: SyntaxDocument) extends StmtParser {
 
-  private var _lambdaNames = collection.mutable.ArrayBuffer[String]()
+  private val _lambdaNames = collection.mutable.ArrayBuffer[String]()
   private var _nameType: NameTokenType.Value = NameTokenType.UNKNOWN
   private var _source: TokenSource = _
 
@@ -25,6 +25,7 @@ class PythonStmtParser(val document: SyntaxDocument) extends StmtParser {
   def parse(source: TokenSource): StatementType =
     if (source != null) {
       _source = source
+      //source.dump()
       parseStmt()
     } else
       null

src/main/scala/tigerjython/syntaxsupport/parser/TokenSource.scala

@@ -7,7 +7,7 @@
  */
 package tigerjython.syntaxsupport.parser
 
-import tigerjython.syntaxsupport.struct.StructElement
+import tigerjython.syntaxsupport.struct.{StructElement, StructLine}
 import tigerjython.syntaxsupport.tokens._
 
 import scala.collection.BufferedIterator
@@ -19,20 +19,35 @@ class TokenSource(val source: TokenArray, val structElement: StructElement) exte
 
   protected val tokens: Array[Token] =
     if (structElement != null && structElement.length > 0) {
-      val idx = structElement.index
+      val idx = structElement.index max 0
       val result = collection.mutable.ArrayBuffer[Token]()
       for (i <- idx until ((idx + structElement.length) min source.length)) {
         val tkn = source(i)
-        if (tkn != null && tkn.tokenType != TokenType.WHITESPACE && tkn.tokenType != TokenType.COMMENT &&
-          tkn.tokenType != TokenType.NEWLINE)
-          result += tkn
+        if (tkn != null) {
+          if (tkn.tokenType == TokenType.NEWLINE) {
+            if (structElement.isInstanceOf[StructLine])
+              result += tkn
+          }
+          else if (tkn.tokenType != TokenType.WHITESPACE && tkn.tokenType != TokenType.COMMENT)
+            result += tkn
+        }
       }
       for (tkn <- result)
         tkn.annotation = false
       result.toArray
     } else
       Array()
 
+  def dump(): Unit = {
+    println("TokenSource:")
+    for ((token, i) <- tokens.zipWithIndex) {
+      if (i == _index)
+        print('|')
+      print(token)
+    }
+    println()
+  }
+
   private var _index: Int = 0
 
   def back(): Unit =

src/main/scala/tigerjython/syntaxsupport/struct/StructContainer.scala

@@ -121,7 +121,7 @@ abstract class StructContainer extends StructElement {
   }
 
   def handleMessage(relIndex: Int, message: TokenChangeMessage): Unit =
-    if (0 <= relIndex && relIndex <= length) {
+    if (0 <= relIndex + offset && relIndex <= length) {
       passMessageToChildren(relIndex, message)
       if (!message.isHandled)
         message match {
@@ -148,9 +148,20 @@ abstract class StructContainer extends StructElement {
             invalidate(tokens)
           case _ =>
         }
-      else if (isDefinitionStatement)
-        invalidate(message.tokens)
-  }
+      else
+        message match {
+          case TokenChangeMessage.TokensDeleted(tokens, _, delCount) if relIndex + delCount < length =>
+            if (relIndex < 0)
+              length -= (relIndex + delCount)
+            else
+              length -= delCount
+            invalidate(tokens)
+          case TokenChangeMessage.TokensInserted(tokens, _, insCount) if relIndex < length =>
+            length += insCount
+            invalidate(tokens)
+          case _ =>
+        }
+    }
 
   protected def handleDeleteMessage(relIndex: Int, tokens: TokenArray, index: Int, delCount: Int): Boolean =
     if (children.nonEmpty)
@@ -175,8 +186,10 @@ abstract class StructContainer extends StructElement {
               case _ =>
                 parse(tokens, index)
             }
-          } else
+          } else {
             parse(tokens, index)
+            length -= delCount
+          }
           true
         case _ =>
           length -= delCount

src/main/scala/tigerjython/syntaxsupport/struct/StructElement.scala

@@ -90,7 +90,7 @@ abstract class StructElement {
     else
       null
 
-  protected def getToken(relIndex: Int): Token =
+  def getToken(relIndex: Int): Token =
     _getToken(this.index + relIndex)
 
   def handleMessage(relIndex: Int, message: TokenChangeMessage): Unit

src/main/scala/tigerjython/syntaxsupport/struct/StructLine.scala

@@ -121,6 +121,11 @@ class StructLine extends StructContainer {
   }
 
   protected def invalidateAstNode(): Unit = {
+    //StructLine.linesWithInvalidatedAstNodes += this
+    validateAstNode()
+  }
+
+  protected def validateAstNode(): Unit = {
     val parser = getParser
     if (parser != null)
       astNode = parser.parse(this)
@@ -163,5 +168,13 @@ class StructLine extends StructContainer {
 }
 object StructLine {
 
+  private val linesWithInvalidatedAstNodes = collection.mutable.Set[StructLine]()
+
   def apply(): StructLine = new StructLine()
+
+  def validateAstNodes(): Unit = {
+    /*for (line <- linesWithInvalidatedAstNodes)
+      line.validateAstNode()
+    linesWithInvalidatedAstNodes.clear()*/
+  }
 }

src/main/scala/tigerjython/syntaxsupport/struct/StructProgram.scala

@@ -63,6 +63,20 @@ class StructProgram(val document: SyntaxDocument) extends StructContainer {
   protected def createParser(): StmtParser =
     new PythonStmtParser(document)
 
+  def dump(): Unit = {
+    println("=" * 70)
+    for ((child, j) <- children.zipWithIndex)
+      child match {
+        case line: StructLine =>
+          println(s"LINE $j: ")
+          for (i <- 0 until line.length)
+            print(line.getToken(i))
+          println()
+        case _ =>
+          println(s"LINE $j: ???")
+      }
+  }
+
   override protected def getDocument: SyntaxDocument = document
 
   def getFirstLineOfBlock(lineNo: Int): Int = {
@@ -100,7 +114,7 @@ class StructProgram(val document: SyntaxDocument) extends StructContainer {
     // Check for the special case where we are on the 'head' line of a composite statement
     if (i == lineNo && i+1 < children.length && this(i+1).indent > indent) {
       i = lineNo + 1
-      val indent = this(i+1).indent
+      val indent = this(i).indent
       while (i < children.length && this(i).hasIndent(indent))
         i += 1
       (getPhysicalLineOf(lineNo), getPhysicalLineOf(i))

src/main/scala/tigerjython/syntaxsupport/struct/TokenChangeMessage.scala

@@ -30,10 +30,21 @@ sealed abstract class TokenChangeMessage {
 object TokenChangeMessage {
 
   // This is fired when a token has changed due to some smaller user input such as a user entering a number or name
-  case class TokenChanged(tokens: TokenArray, index: Int) extends TokenChangeMessage
+  case class TokenChanged(tokens: TokenArray, index: Int) extends TokenChangeMessage {
+
+    override def toString: String = {
+      s"TokenChanged($index, '${tokens(index)}')"
+    }
+  }
 
   // This is fired when a part of the text was deleted
-  case class TokensDeleted(tokens: TokenArray, index: Int, delCount: Int) extends TokenChangeMessage
+  case class TokensDeleted(tokens: TokenArray, index: Int, delCount: Int) extends TokenChangeMessage {
+
+    override def toString: String = {
+      val tkns = tokens.slice((index-1) max 0, (index+5) min tokens.length).mkString(", ")
+      s"TokensDeleted($index, $delCount, '$tkns')"
+    }
+  }
 
   // This is fired when new text was inserted
   case class TokensInserted(tokens: TokenArray, index: Int, insCount: Int) extends TokenChangeMessage {
@@ -51,5 +62,10 @@ object TokenChangeMessage {
         }
       (brackets, newlines)
     }
+
+    override def toString: String = {
+      val tkns = tokens.slice((index-1) max 0, (index+insCount+4) min tokens.length).take(12).mkString(", ")
+      s"TokensInserted($index, $insCount, '$tkns')"
+    }
   }
 }

src/main/scala/tigerjython/syntaxsupport/tokens/NameToken.scala

@@ -7,7 +7,6 @@
  */
 package tigerjython.syntaxsupport.tokens
 
-import tigerjython.syntaxsupport.{SyntaxDocument, TokenVisitor}
 import tigerjython.syntaxsupport.struct.NameInfo
 
 /**
@@ -17,8 +16,15 @@ class NameToken(tt: TokenType.Value, private var _text: String) extends Token(tt
 
   private var _isStmtOnlyKeyword: Boolean = false
 
+  private var _nameTokenType: NameTokenType.Value = NameTokenType.UNKNOWN
+
   var nameInfo: NameInfo = _
-  var nameTokenType: NameTokenType.Value = NameTokenType.UNKNOWN
+  // var nameTokenType: NameTokenType.Value = NameTokenType.UNKNOWN
+
+  def nameTokenType: NameTokenType.Value = _nameTokenType
+  def nameTokenType_=(ntt: NameTokenType.Value): Unit = {
+    _nameTokenType = ntt
+  }
 
   override protected def getStyleName: String =
     tokenType match {