moo icon indicating copy to clipboard operation
moo copied to clipboard

词法定义中大小写忽略与关键字字面量处理问题

Open cnsource opened this issue 6 months ago • 0 comments

    // 忽略关键字大小写的关键函数,支持语法定义中直接使用关键字字面量
    function makeCaseInsensitive(lexer, tokenType) {

        // 将tokenType 转换成map,
        const map = new Map();
        for (const [type, keywords] of Object.entries(tokenType)) {
            Array.from(keywords).forEach(k => map.set(k.toLowerCase(), type));
        }

        lexer.next = (next => () => {
            const token = next.call(lexer);
            if (token && (token.text || token.value)) {
                let lowerText = null;

                if (token.text) {
                    lowerText = token.text.toLowerCase();
                } else if (token.value) {
                    lowerText = token.value.toLowerCase();
                }

                if (map.has(lowerText)) {
                    token.text = lowerText;
                    token.value = lowerText;
                    token.type = map.get(lowerText);
                }
            }
            return token;
        })(lexer.next);
        return lexer;
    }

    function createLexer(tokenType) {

        const map = new Map();
        for (const [type, keywords] of Object.entries(tokenType)) {
            Array.from(keywords).forEach(k => map.set(k.toLowerCase(), type));
        }

        const moo = require("moo");
        const lexer = moo.compile({
          ws: /[ \t]+/,
          nl: { match: /[\n\r]+/, lineBreaks: true },
          // 标识符
          IDEN: {
              match: /[a-zA-Z_][a-zA-Z0-9_]*/,
              type: moo.keywords(Object.fromEntries(map))
          },
      });

      return lexer;
   }

    const tokenType = {
        KW: new Set([
            "define",
        ]),
        Marco: new Set([

        ]),
    }


    const lexer = createLexer(tokenType);
    makeCaseInsensitive(lexer, tokenType);

cnsource avatar Oct 25 '25 02:10 cnsource