Skip to content

词法定义中大小写忽略与关键字字面量处理问题 #187

@cnsource

Description

@cnsource
    // 忽略关键字大小写的关键函数,支持语法定义中直接使用关键字字面量
    function makeCaseInsensitive(lexer, tokenType) {

        // 将tokenType 转换成map,
        const map = new Map();
        for (const [type, keywords] of Object.entries(tokenType)) {
            Array.from(keywords).forEach(k => map.set(k.toLowerCase(), type));
        }

        lexer.next = (next => () => {
            const token = next.call(lexer);
            if (token && (token.text || token.value)) {
                let lowerText = null;

                if (token.text) {
                    lowerText = token.text.toLowerCase();
                } else if (token.value) {
                    lowerText = token.value.toLowerCase();
                }

                if (map.has(lowerText)) {
                    token.text = lowerText;
                    token.value = lowerText;
                    token.type = map.get(lowerText);
                }
            }
            return token;
        })(lexer.next);
        return lexer;
    }

    function createLexer(tokenType) {

        const map = new Map();
        for (const [type, keywords] of Object.entries(tokenType)) {
            Array.from(keywords).forEach(k => map.set(k.toLowerCase(), type));
        }

        const moo = require("moo");
        const lexer = moo.compile({
          ws: /[ \t]+/,
          nl: { match: /[\n\r]+/, lineBreaks: true },
          // 标识符
          IDEN: {
              match: /[a-zA-Z_][a-zA-Z0-9_]*/,
              type: moo.keywords(Object.fromEntries(map))
          },
      });

      return lexer;
   }

    const tokenType = {
        KW: new Set([
            "define",
        ]),
        Marco: new Set([

        ]),
    }


    const lexer = createLexer(tokenType);
    makeCaseInsensitive(lexer, tokenType);

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions