Skip to content

Commit

Permalink
Use a proper js parser to check for syntax errors
Browse files Browse the repository at this point in the history 
- Adds support for multiple functions
- Adds line numbers for more syntax errors
  • Loading branch information
@varun7654
varun7654 committed May 9, 2024
1 parent 57515ce commit b13e5dc
Show file tree
Hide file tree
Showing 4 changed files with 132 additions and 169 deletions.
2 changes: 2 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,8 @@ The solution is given to the AI assistant to help it provide hints to the user,

It is additionally used to determine the correct answers to the test cases.

*The function name that should be called must be the first method in the code block. (if there are multiple methods)*

### Test Cases (required)

If you don't want to provide any visible test cases, you can leave this section empty, but it must be present.
Expand Down
13 changes: 13 additions & 0 deletions package-lock.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
"@types/react": "^18.2.69",
"@types/react-dom": "^18.3.0",
"ace-builds": "^1.32.7",
"acorn": "^8.11.3",
"acorn-loose": "^8.4.0",
"capture-console-logs": "^2.0.1-rc.1",
"dompurify": "^3.0.9",
"eslint": "^8.57.0",
Expand Down
284 changes: 115 additions & 169 deletions src/problem/CodeRunner.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@ import {UserData} from "./Problem";
import {ProblemData} from "./ProblemParse";
import {Log} from "capture-console-logs/dist/logs";
import * as util from "util";
import * as acorn from "acorn";

const acornLoose = require("acorn-loose");

const functionHeaderOffset = 2;

Expand Down Expand Up @@ -158,204 +161,127 @@ function safeToString(expectedResult: any) {
export function testUserCode(userData: UserData, problemData: ProblemData): TestResults {
let userCode = userData.currentCode;

// Check that we have balanced brackets
{
let brackets = 0;
let doubleQuotes = false;
let singleQuotes = false;
let backticks = false;
let lineNum = 1;
let foundFirstBracket = false;
let whitespaceRegex = /^\s*$/;
let characterAfterLastBracket = {value: false, lineNum: -1};
for (let i = 0; i < userCode.length; i++) {

if (userCode[i] === '"' && !singleQuotes && !backticks) {
doubleQuotes = !doubleQuotes;
}
if (userCode[i] === "'" && !doubleQuotes && !backticks) {
singleQuotes = !singleQuotes;
}
if (userCode[i] === "`" && !doubleQuotes && !singleQuotes) {
backticks = !backticks;
}
if (doubleQuotes || singleQuotes || backticks) {
continue;
}
if (brackets === 0 && foundFirstBracket && !userCode[i].match(whitespaceRegex) && !characterAfterLastBracket.value) {
characterAfterLastBracket = {value: true, lineNum: lineNum};
}

if (userCode[i] === '{') {
if (foundFirstBracket && brackets === 0) {
return {
testResults: [],
expectedResults: getExpectedResults(problemData),
returnedResults: [],
parseError: "You began a new function after closing your first one. You cannot do that. If you want to define a new function, do it inside the first function.",
errorLine: lineNum,
runtimeError: "",
outputs: [],
ranSuccessfully: false
};
}

brackets++;
foundFirstBracket = true;
} else if (userCode[i] === '}') {
brackets--;
}

if (userCode[i] === '\n') {
lineNum++;
}

if (brackets < 0) {
return {
testResults: [],
returnedResults: [],
expectedResults: getExpectedResults(problemData),
parseError: "Unbalanced brackets. Extra '}' found.",
errorLine: lineNum,
runtimeError: "",
outputs: [],
ranSuccessfully: false
};
}
}

if (doubleQuotes || singleQuotes || backticks) {
let type = doubleQuotes ? "double quotes" : singleQuotes ? "single quotes" : "backticks";
let ast;
try {
ast = acorn.parse(userCode, {ecmaVersion: "latest", locations: true});
} catch (e) {
if (e instanceof SyntaxError) {
return {
testResults: [],
returnedResults: [],
expectedResults: getExpectedResults(problemData),
parseError: "Unbalanced " + type + ". Missing closing " + type + ".",
errorLine: lineNum,
parseError: e.message,
// @ts-ignore
errorLine: e.loc.line,
runtimeError: "",
outputs: [],
ranSuccessfully: false
};
} else {
throw e;
}
}

if (brackets !== 0) {
return {
testResults: [],
returnedResults: [],
expectedResults: getExpectedResults(problemData),
parseError: "Unbalanced brackets. Missing '}'.",
errorLine: lineNum,
runtimeError: "",
outputs: [],
ranSuccessfully: false
};
}

if (!foundFirstBracket) {
return {
{
let missingFunctionError = {
returnableError: {
testResults: [],
returnedResults: [],
expectedResults: getExpectedResults(problemData),
parseError: "No function found. We expected to see at least one pair of '{}'.",
errorLine: lineNum,
parseError: "You need to define a function with the following signature:" + problemData.solutionCode.split('{')[0],
errorLine: 1,
runtimeError: "",
outputs: [],
ranSuccessfully: false
};
}
},
matchedTokens: 0,
// The levenshteinDistance between the missed token
levenshteinDistance: 100000
};

if (characterAfterLastBracket.value) {
return {
testResults: [],
returnedResults: [],
expectedResults: getExpectedResults(problemData),
parseError: "You have stray character(s) after the last '}'.",
errorLine: characterAfterLastBracket.lineNum,
runtimeError: "",
outputs: [],
ranSuccessfully: false
};
}
}
let foundFunction = false;

// Check that the function signature is correct
{
let functionSignature = userCode.split('{')[0];
let tokens = tokenizeFunctionSignature(functionSignature);
fnLoop: for (let func of ast.body) {
let functionSignature = userCode.substring(func.start, func.end).split('{')[0];
let tokens = tokenizeFunctionSignature(functionSignature);

let expectedFunctionSignature = problemData.solutionCode.split('{')[0];
let expectedTokens = tokenizeFunctionSignature(expectedFunctionSignature);
let expectedFunctionSignature = problemData.solutionCode.split('{')[0];
let expectedTokens = tokenizeFunctionSignature(expectedFunctionSignature);

for (let i = 0; i < tokens.length; i++) {
if (tokens[i].str !== expectedTokens[i].str) {
for (let i = 0; i < tokens.length; i++) {
if (tokens[i].str !== expectedTokens[i].str) {

let parseError = "Function signature does not match the expected signature. ";
if (i === 0) {
parseError += "\nThe function signature should begin with `" + expectedTokens[i].str + "` but you have ";
if (tokens[i] === undefined || tokens[i].str === "") {
parseError += "nothing.";
let parseError = "Function signature does not match the expected signature. ";
if (i === 0) {
parseError += "\nThe function signature should begin with `" + expectedTokens[i].str + "` but you have ";
if (tokens[i] === undefined || tokens[i].str === "") {
parseError += "nothing.";
} else {
parseError += "`" + tokens[i].str + "`.";
}
} else {
parseError += "`" + tokens[i].str + "`.";
if (tokens[i] === undefined || tokens[i].str === "") {
parseError += "Expected: `" + expectedTokens[i].str + "` but got nothing.";
} else {
parseError += "Expected: `" + expectedTokens[i].str + "` after `" + tokens.slice(0, i)
.map(t => t.str).join(" ") + "` but got: `" + tokens[i].str + "`.";
}
}
} else {
if (tokens[i] === undefined || tokens[i].str === "") {
parseError += "Expected: `" + expectedTokens[i].str + "` but got nothing.";
} else {
parseError += "Expected: `" + expectedTokens[i].str + "` after `" + tokens.slice(0, i)
.map(t => t.str).join(" ") + "` but got: `" + tokens[i].str + "`.";

let distance = levenshteinDistance(tokens[i].str, expectedTokens[i].str);

// We also check the levenshtein distance
// to see if the user has a typo and put the error on the closest match
if (i > missingFunctionError.matchedTokens ||
(distance < missingFunctionError.levenshteinDistance && i >= missingFunctionError.matchedTokens)) {
missingFunctionError = {
returnableError: {
testResults: [],
returnedResults: [],
expectedResults: getExpectedResults(problemData),
parseError,
errorLine: tokens[i].lineNum,
runtimeError: "",
outputs: [],
ranSuccessfully: false
},
matchedTokens: i,
levenshteinDistance: distance
}
}
continue fnLoop;
}
return {
testResults: [],
returnedResults: [],
expectedResults: getExpectedResults(problemData),
parseError,
errorLine: tokens[i].lineNum,
runtimeError: "",
outputs: [],
ranSuccessfully: false
};
}
}

if (tokens.length !== expectedTokens.length) {
return {
testResults: [],
returnedResults: [],
expectedResults: getExpectedResults(problemData),
parseError: "Function signature does not match the expected signature. " +
"Expected: " + expectedFunctionSignature + " but got: " + functionSignature,
errorLine: tokens[tokens.length - 1].lineNum,
runtimeError: "",
outputs: [],
ranSuccessfully: false
};
if (tokens.length !== expectedTokens.length) {
if (tokens.length > missingFunctionError.matchedTokens) {
missingFunctionError = {
returnableError: {
testResults: [],
returnedResults: [],
expectedResults: getExpectedResults(problemData),
parseError: "Function signature does not match the expected signature. " +
"Expected: " + expectedFunctionSignature + " but got: " + functionSignature,
errorLine: tokens[tokens.length - 1].lineNum,
runtimeError: "",
outputs: [],
ranSuccessfully: false
},
matchedTokens: tokens.length,
levenshteinDistance: 100000
}
}
continue;
}

foundFunction = true;
break;
}
}

// Try putting the user's code in a function and running it to catch syntax errors
//eslint-disable-next-line
let testUserCode = `
${userCode}
`;
try {
// eslint-disable-next-line
let func = Function(testUserCode);
func();
} catch (e) {
console.error("Failed to run the user's code: " + e);
let error = e as Error;
console.log(error.stack);
return {
testResults: [],
returnedResults: [],
expectedResults: getExpectedResults(problemData),
parseError: error.toString(),
errorLine: -1,
runtimeError: "",
outputs: [],
ranSuccessfully: false
};
if (!foundFunction) {
return missingFunctionError.returnableError;
}
}

// We need to look for all the loops (for, while, do-while) and insert code to count the number of iterations.
Expand Down Expand Up @@ -676,4 +602,24 @@ ${solutionCode}
}

return expectedResultsArray.map(result => safeToString(result));
}
}

const levenshteinDistance = (s: string, t: string) => {
if (!s.length) return t.length;
if (!t.length) return s.length;
const arr = [];
for (let i = 0; i <= t.length; i++) {
arr[i] = [i];
for (let j = 1; j <= s.length; j++) {
arr[i][j] =
i === 0
? j
: Math.min(
arr[i - 1][j] + 1,
arr[i][j - 1] + 1,
arr[i - 1][j - 1] + (s[j - 1] === t[i - 1] ? 0 : 1)
);
}
}
return arr[t.length][s.length];
};

0 comments on commit b13e5dc

Please sign in to comment.