Merge remote-tracking branch 'ups/dev' into force-compile

Author: georgecao

.github/workflows/hosted.yml

@@ -21,7 +21,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [
-          macos-11,
+          macos-12,
           ubuntu-20.04,
           windows-2022
         ]
@@ -160,7 +160,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [
-          macos-11,
+          macos-12,
           ubuntu-20.04,
           windows-2022
         ]
@@ -196,6 +196,14 @@ jobs:
         repository: antlr/antlr-php-runtime
         path: runtime/PHP
 
+    - name: Setup PHP 8.2
+      if: matrix.target == 'php'
+      uses: shivammathur/setup-php@v2
+      with:
+        php-version: '8.3'
+        extensions: mbstring
+        tools: composer
+
     - name: Install dependencies
       env:
         COMPOSER_CACHE_DIR: ${{ github.workspace }}/.cache
@@ -263,14 +271,6 @@ jobs:
       with:
         go-version: '^1.19'
 
-    - name: Setup PHP 8.2
-      if: matrix.target == 'php'
-      uses: shivammathur/[email protected]
-      with:
-        php-version: '8.2'
-        extensions: mbstring
-        tools: composer
-
     - name: Setup Swift
       if: matrix.target == 'swift'
       uses: swift-actions/[email protected]

README.md

@@ -22,7 +22,7 @@
 ## Versioning
 
 ANTLR 4 supports 10 target languages
-(Cpp, CSharp, Dart, Java, JavaScript, PHP, Python3, Swift, TypeScript),
+(Cpp, CSharp, Dart, Java, JavaScript, PHP, Python3, Swift, TypeScript, Go),
 and ensuring consistency across these targets is a unique and highly valuable feature.
 To ensure proper support of this feature, each release of ANTLR is a complete release of the tool and the 10 runtimes, all with the same version.
 As such, ANTLR versioning does not strictly follow semver semantics:
@@ -54,7 +54,7 @@ The dedicated Go repo is for `go get` and `import` only. Go runtime development
 ANTLR project lead and supreme dictator for life
 [University of San Francisco](http://www.usfca.edu/)
 * [Sam Harwell](http://tunnelvisionlabs.com/) (Tool co-author, Java and original C# target)
-* [Eric Vergnaud](https://github.com/ericvergnaud) (Javascript, Python2, Python3 targets and maintenance of C# target)
+* [Eric Vergnaud](https://github.com/ericvergnaud) (Javascript, TypeScript, Python2, Python3 targets and maintenance of C# target)
 * [Peter Boyer](https://github.com/pboyer) (Go target)
 * [Mike Lischke](http://www.soft-gems.net/) (C++ completed target)
 * Dan McLaughlin (C++ initial target)
@@ -77,7 +77,7 @@ ANTLR project lead and supreme dictator for life
 * [Official site](http://www.antlr.org/)
 * [Documentation](https://github.com/antlr/antlr4/blob/master/doc/index.md)
 * [FAQ](https://github.com/antlr/antlr4/blob/master/doc/faq/index.md)
-* [ANTLR code generation targets](https://github.com/antlr/antlr4/blob/master/doc/targets.md)<br>(Currently: Java, C#, Python3, JavaScript, Go, C++, Swift, Dart, PHP)
+* [ANTLR code generation targets](https://github.com/antlr/antlr4/blob/master/doc/targets.md)<br>(Currently: Java, C#, Python3, JavaScript, TypeScript, Go, C++, Swift, Dart, PHP)
 * _Note: As of version 4.14, we are dropping support for Python 2. We love the Python
 community, but Python 2 support was officially halted in Jan 2020. More recently,
 GiHub also dropped support for Python 2, which has made it impossible for us to

doc/javascript-target.md

@@ -40,8 +40,6 @@ Once you've generated the lexer and/or parser code, you need to download the run
 
 The JavaScript runtime is [available from npm](https://www.npmjs.com/package/antlr4).
 
-If you can't use npm, the JavaScript runtime is also available from the ANTLR web site [download section](http://www.antlr.org/download/index.html). The runtime is provided in the form of source code, so no additional installation is required.
-
 We will not document here how to refer to the runtime from your project, since this would differ a lot depending on your project type and IDE. 
 
 ## How do I get the runtime in my browser?

doc/python-target.md

@@ -50,7 +50,7 @@ must include a reference to the Antlr Python3 runtime.
 You should provide a script that builds the program.
     ```
     pip install -r requirements.txt
-    antlr4 -v 4.13.0 -Dlanguage=Python3 Expr.g4
+    antlr4 -Dlanguage=Python3 Expr.g4
     ```
 _It is vital that the versions for the
 Antlr tool used to generate the parser

runtime/Cpp/cmake/README.md

@@ -34,7 +34,7 @@ set(ANTLR4_WITH_STATIC_CRT OFF)
 
 # add external build for antlrcpp
 include(ExternalAntlr4Cpp)
-# add antrl4cpp artifacts to project environment
+# add antlr4cpp artifacts to project environment
 include_directories(${ANTLR4_INCLUDE_DIRS})
 
 # set variable pointing to the antlr tool that supports C++

runtime/Cpp/runtime/src/TokenStreamRewriter.cpp

@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
  * Use of this file is governed by the BSD 3-clause license that
  * can be found in the LICENSE.txt file in the project root.
  */
@@ -313,6 +313,10 @@ std::string TokenStreamRewriter::getText(const std::string &programName, const I
 std::unordered_map<size_t, TokenStreamRewriter::RewriteOperation*> TokenStreamRewriter::reduceToSingleOperationPerIndex(
   std::vector<TokenStreamRewriter::RewriteOperation*> &rewrites) {
 
+  // Reset the instructionIndex
+  for (size_t i = 0; i < rewrites.size(); ++i) {
+    rewrites[i]->instructionIndex = i;
+  }
 
   // WALK REPLACES
   for (size_t i = 0; i < rewrites.size(); ++i) {
@@ -327,35 +331,34 @@ std::unordered_map<size_t, TokenStreamRewriter::RewriteOperation*> TokenStreamRe
       if (iop->index == rop->index) {
         // E.g., insert before 2, delete 2..2; update replace
         // text to include insert before, kill insert
-        delete rewrites[iop->instructionIndex];
-        rewrites[iop->instructionIndex] = nullptr;
         rop->text = iop->text + (!rop->text.empty() ? rop->text : "");
+        rewrites[iop->instructionIndex] = nullptr;
+        delete iop;
       }
       else if (iop->index > rop->index && iop->index <= rop->lastIndex) {
         // delete insert as it's a no-op.
-        delete rewrites[iop->instructionIndex];
         rewrites[iop->instructionIndex] = nullptr;
+        delete iop;
       }
     }
     // Drop any prior replaces contained within
     std::vector<ReplaceOp*> prevReplaces = getKindOfOps<ReplaceOp>(rewrites, i);
     for (auto *prevRop : prevReplaces) {
       if (prevRop->index >= rop->index && prevRop->lastIndex <= rop->lastIndex) {
         // delete replace as it's a no-op.
-        delete rewrites[prevRop->instructionIndex];
         rewrites[prevRop->instructionIndex] = nullptr;
+        delete prevRop;
         continue;
       }
       // throw exception unless disjoint or identical
       bool disjoint = prevRop->lastIndex < rop->index || prevRop->index > rop->lastIndex;
       // Delete special case of replace (text==null):
       // D.i-j.u D.x-y.v    | boundaries overlap    combine to max(min)..max(right)
       if (prevRop->text.empty() && rop->text.empty() && !disjoint) {
-        delete rewrites[prevRop->instructionIndex];
-        rewrites[prevRop->instructionIndex] = nullptr; // kill first delete
         rop->index = std::min(prevRop->index, rop->index);
         rop->lastIndex = std::max(prevRop->lastIndex, rop->lastIndex);
-        std::cout << "new rop " << rop << std::endl;
+        rewrites[prevRop->instructionIndex] = nullptr; // kill first delete
+        delete prevRop;
       }
       else if (!disjoint) {
         throw IllegalArgumentException("replace op boundaries of " + rop->toString() +
@@ -379,8 +382,8 @@ std::unordered_map<size_t, TokenStreamRewriter::RewriteOperation*> TokenStreamRe
                                           // whole token buffer so no lazy eval issue with any templates
         iop->text = catOpText(&iop->text, &prevIop->text);
         // delete redundant prior insert
-        delete rewrites[prevIop->instructionIndex];
         rewrites[prevIop->instructionIndex] = nullptr;
+        delete prevIop;
       }
     }
     // look for replaces where iop.index is in range; error

runtime/Go/antlr/README.adoc

@@ -10,7 +10,7 @@ Your driver code etc. should now be importing from the new release only repo for
 
 ```go
 import (
- github.com/antlr4-go/antlr
+	"github.com/antlr4-go/antlr"
 )
 ```
 

runtime/Go/antlr/go.sum

@@ -4,8 +4,6 @@
 
 package antlr
 
-import "sync"
-
 // ATNInvalidAltNumber is used to represent an ALT number that has yet to be calculated or
 // which is invalid for a particular struct such as [*antlr.BaseRuleContext]
 var ATNInvalidAltNumber int
@@ -56,9 +54,9 @@ type ATN struct {
 	//
 	states []ATNState
 
-	mu      sync.Mutex
-	stateMu sync.RWMutex
-	edgeMu  sync.RWMutex
+	mu      Mutex
+	stateMu RWMutex
+	edgeMu  RWMutex
 }
 
 // NewATN returns a new ATN struct representing the given grammarType and is used

runtime/Go/antlr/v4/atn.go

@@ -73,9 +73,6 @@ func NewATNConfig1(c *ATNConfig, state ATNState, context *PredictionContext) *AT
 // NewATNConfig creates a new ATNConfig instance given an existing config, a state, a context and a semantic context, other 'constructors'
 // are just wrappers around this one.
 func NewATNConfig(c *ATNConfig, state ATNState, context *PredictionContext, semanticContext SemanticContext) *ATNConfig {
-	if semanticContext == nil {
-		panic("semanticContext cannot be nil") // TODO: Remove this - probably put here for some bug that is now fixed
-	}
 	b := &ATNConfig{}
 	b.InitATNConfig(c, state, c.GetAlt(), context, semanticContext)
 	b.cType = parserConfig

runtime/Go/antlr/v4/atn_config.go

@@ -8,7 +8,6 @@ import (
 	"container/list"
 	"runtime/debug"
 	"sort"
-	"sync"
 )
 
 // Collectable is an interface that a struct should implement if it is to be
@@ -587,12 +586,12 @@ type VisitRecord struct {
 
 type VisitList struct {
 	cache *list.List
-	lock  sync.RWMutex
+	lock  RWMutex
 }
 
 var visitListPool = VisitList{
 	cache: list.New(),
-	lock:  sync.RWMutex{},
+	lock:  RWMutex{},
 }
 
 // NewVisitRecord returns a new VisitRecord instance from the pool if available.

runtime/Go/antlr/v4/jcollect.go

@@ -40,6 +40,7 @@ func (la *LL1Analyzer) getDecisionLookahead(s ATNState) []*IntervalSet {
 	for alt := 0; alt < count; alt++ {
 
 		look[alt] = NewIntervalSet()
+		// TODO: This is one of the reasons that ATNConfigs are allocated and freed all the time - fix this tomorrow jim!
 		lookBusy := NewJStore[*ATNConfig, Comparator[*ATNConfig]](aConfEqInst, ClosureBusyCollection, "LL1Analyzer.getDecisionLookahead for lookBusy")
 		la.look1(s.GetTransitions()[alt].getTarget(), nil, BasePredictionContextEMPTY, look[alt], lookBusy, NewBitSet(), false, false)
 

runtime/Go/antlr/v4/ll1_analyzer.go

@@ -0,0 +1,41 @@
+//go:build !antlr.nomutex
+// +build !antlr.nomutex
+
+package antlr
+
+import "sync"
+
+// Mutex is a simple mutex implementation which just delegates to sync.Mutex, it
+// is used to provide a mutex implementation for the antlr package, which users
+// can turn off with the build tag -tags antlr.nomutex
+type Mutex struct {
+	mu sync.Mutex
+}
+
+func (m *Mutex) Lock() {
+	m.mu.Lock()
+}
+
+func (m *Mutex) Unlock() {
+	m.mu.Unlock()
+}
+
+type RWMutex struct {
+	mu sync.RWMutex
+}
+
+func (m *RWMutex) Lock() {
+	m.mu.Lock()
+}
+
+func (m *RWMutex) Unlock() {
+	m.mu.Unlock()
+}
+
+func (m *RWMutex) RLock() {
+	m.mu.RLock()
+}
+
+func (m *RWMutex) RUnlock() {
+	m.mu.RUnlock()
+}

runtime/Go/antlr/v4/mutex.go

@@ -0,0 +1,31 @@
+// +build antlr.nomutex
+
+package antlr
+
+type Mutex struct{}
+
+func (m *Mutex) Lock() {
+	// No-op
+}
+
+func (m *Mutex) Unlock() {
+	// No-op
+}
+
+type RWMutex struct{}
+
+func (m *RWMutex) Lock() {
+	// No-op
+}
+
+func (m *RWMutex) Unlock() {
+	// No-op
+}
+
+func (m *RWMutex) RLock() {
+	// No-op
+}
+
+func (m *RWMutex) RUnlock() {
+	// No-op
+}

runtime/Go/antlr/v4/mutex_nomutex.go

@@ -6,7 +6,6 @@ package antlr
 
 import (
 	"fmt"
-	"golang.org/x/exp/slices"
 	"strconv"
 )
 
@@ -115,6 +114,9 @@ func (p *PredictionContext) Hash() int {
 }
 
 func (p *PredictionContext) Equals(other Collectable[*PredictionContext]) bool {
+	if p == other {
+		return true
+	}
 	switch p.pcType {
 	case PredictionContextEmpty:
 		otherP := other.(*PredictionContext)
@@ -141,18 +143,16 @@ func (p *PredictionContext) ArrayEquals(o Collectable[*PredictionContext]) bool
 
 	// Must compare the actual array elements and not just the array address
 	//
-	return slices.Equal(p.returnStates, other.returnStates) &&
-		slices.EqualFunc(p.parents, other.parents, func(x, y *PredictionContext) bool {
-			return x.Equals(y)
-		})
+	return intSlicesEqual(p.returnStates, other.returnStates) &&
+		pcSliceEqual(p.parents, other.parents)
 }
 
 func (p *PredictionContext) SingletonEquals(other Collectable[*PredictionContext]) bool {
 	if other == nil {
 		return false
 	}
 	otherP := other.(*PredictionContext)
-	if otherP == nil {
+	if otherP == nil || otherP.pcType != PredictionContextSingleton {
 		return false
 	}
 

runtime/Go/antlr/v4/prediction_context.go

@@ -9,7 +9,6 @@ import (
 	"path/filepath"
 	"sort"
 	"strconv"
-	"sync"
 )
 
 // This file allows the user to collect statistics about the runtime of the ANTLR runtime. It is not enabled by default
@@ -30,7 +29,7 @@ type goRunStats struct {
 	// within this package.
 	//
 	jStats            []*JStatRec
-	jStatsLock        sync.RWMutex
+	jStatsLock        RWMutex
 	topN              int
 	topNByMax         []*JStatRec
 	topNByUsed        []*JStatRec