From 7a52ddeba2a68388b544f529d2d92104420f77b0 Mon Sep 17 00:00:00 2001 From: Shipwreckt Date: Fri, 31 Oct 2025 20:02:14 +0000 Subject: Changed from static to 11ty! --- node_modules/nunjucks/src/lexer.js | 474 +++++++++++++++++++++++++++++++++++++ 1 file changed, 474 insertions(+) create mode 100644 node_modules/nunjucks/src/lexer.js (limited to 'node_modules/nunjucks/src/lexer.js') diff --git a/node_modules/nunjucks/src/lexer.js b/node_modules/nunjucks/src/lexer.js new file mode 100644 index 0000000..ca4454d --- /dev/null +++ b/node_modules/nunjucks/src/lexer.js @@ -0,0 +1,474 @@ +'use strict'; + +var lib = require('./lib'); +var whitespaceChars = " \n\t\r\xA0"; +var delimChars = '()[]{}%*-+~/#,:|.<>=!'; +var intChars = '0123456789'; +var BLOCK_START = '{%'; +var BLOCK_END = '%}'; +var VARIABLE_START = '{{'; +var VARIABLE_END = '}}'; +var COMMENT_START = '{#'; +var COMMENT_END = '#}'; +var TOKEN_STRING = 'string'; +var TOKEN_WHITESPACE = 'whitespace'; +var TOKEN_DATA = 'data'; +var TOKEN_BLOCK_START = 'block-start'; +var TOKEN_BLOCK_END = 'block-end'; +var TOKEN_VARIABLE_START = 'variable-start'; +var TOKEN_VARIABLE_END = 'variable-end'; +var TOKEN_COMMENT = 'comment'; +var TOKEN_LEFT_PAREN = 'left-paren'; +var TOKEN_RIGHT_PAREN = 'right-paren'; +var TOKEN_LEFT_BRACKET = 'left-bracket'; +var TOKEN_RIGHT_BRACKET = 'right-bracket'; +var TOKEN_LEFT_CURLY = 'left-curly'; +var TOKEN_RIGHT_CURLY = 'right-curly'; +var TOKEN_OPERATOR = 'operator'; +var TOKEN_COMMA = 'comma'; +var TOKEN_COLON = 'colon'; +var TOKEN_TILDE = 'tilde'; +var TOKEN_PIPE = 'pipe'; +var TOKEN_INT = 'int'; +var TOKEN_FLOAT = 'float'; +var TOKEN_BOOLEAN = 'boolean'; +var TOKEN_NONE = 'none'; +var TOKEN_SYMBOL = 'symbol'; +var TOKEN_SPECIAL = 'special'; +var TOKEN_REGEX = 'regex'; +function token(type, value, lineno, colno) { + return { + type: type, + value: value, + lineno: lineno, + colno: colno + }; +} +var Tokenizer = /*#__PURE__*/function () { + function Tokenizer(str, opts) { + this.str = str; + this.index = 0; + this.len = str.length; + this.lineno = 0; + this.colno = 0; + this.in_code = false; + opts = opts || {}; + var tags = opts.tags || {}; + this.tags = { + BLOCK_START: tags.blockStart || BLOCK_START, + BLOCK_END: tags.blockEnd || BLOCK_END, + VARIABLE_START: tags.variableStart || VARIABLE_START, + VARIABLE_END: tags.variableEnd || VARIABLE_END, + COMMENT_START: tags.commentStart || COMMENT_START, + COMMENT_END: tags.commentEnd || COMMENT_END + }; + this.trimBlocks = !!opts.trimBlocks; + this.lstripBlocks = !!opts.lstripBlocks; + } + var _proto = Tokenizer.prototype; + _proto.nextToken = function nextToken() { + var lineno = this.lineno; + var colno = this.colno; + var tok; + if (this.in_code) { + // Otherwise, if we are in a block parse it as code + var cur = this.current(); + if (this.isFinished()) { + // We have nothing else to parse + return null; + } else if (cur === '"' || cur === '\'') { + // We've hit a string + return token(TOKEN_STRING, this._parseString(cur), lineno, colno); + } else if (tok = this._extract(whitespaceChars)) { + // We hit some whitespace + return token(TOKEN_WHITESPACE, tok, lineno, colno); + } else if ((tok = this._extractString(this.tags.BLOCK_END)) || (tok = this._extractString('-' + this.tags.BLOCK_END))) { + // Special check for the block end tag + // + // It is a requirement that start and end tags are composed of + // delimiter characters (%{}[] etc), and our code always + // breaks on delimiters so we can assume the token parsing + // doesn't consume these elsewhere + this.in_code = false; + if (this.trimBlocks) { + cur = this.current(); + if (cur === '\n') { + // Skip newline + this.forward(); + } else if (cur === '\r') { + // Skip CRLF newline + this.forward(); + cur = this.current(); + if (cur === '\n') { + this.forward(); + } else { + // Was not a CRLF, so go back + this.back(); + } + } + } + return token(TOKEN_BLOCK_END, tok, lineno, colno); + } else if ((tok = this._extractString(this.tags.VARIABLE_END)) || (tok = this._extractString('-' + this.tags.VARIABLE_END))) { + // Special check for variable end tag (see above) + this.in_code = false; + return token(TOKEN_VARIABLE_END, tok, lineno, colno); + } else if (cur === 'r' && this.str.charAt(this.index + 1) === '/') { + // Skip past 'r/'. + this.forwardN(2); + + // Extract until the end of the regex -- / ends it, \/ does not. + var regexBody = ''; + while (!this.isFinished()) { + if (this.current() === '/' && this.previous() !== '\\') { + this.forward(); + break; + } else { + regexBody += this.current(); + this.forward(); + } + } + + // Check for flags. + // The possible flags are according to https://developer.mozilla.org/en/docs/Web/JavaScript/Reference/Global_Objects/RegExp) + var POSSIBLE_FLAGS = ['g', 'i', 'm', 'y']; + var regexFlags = ''; + while (!this.isFinished()) { + var isCurrentAFlag = POSSIBLE_FLAGS.indexOf(this.current()) !== -1; + if (isCurrentAFlag) { + regexFlags += this.current(); + this.forward(); + } else { + break; + } + } + return token(TOKEN_REGEX, { + body: regexBody, + flags: regexFlags + }, lineno, colno); + } else if (delimChars.indexOf(cur) !== -1) { + // We've hit a delimiter (a special char like a bracket) + this.forward(); + var complexOps = ['==', '===', '!=', '!==', '<=', '>=', '//', '**']; + var curComplex = cur + this.current(); + var type; + if (lib.indexOf(complexOps, curComplex) !== -1) { + this.forward(); + cur = curComplex; + + // See if this is a strict equality/inequality comparator + if (lib.indexOf(complexOps, curComplex + this.current()) !== -1) { + cur = curComplex + this.current(); + this.forward(); + } + } + switch (cur) { + case '(': + type = TOKEN_LEFT_PAREN; + break; + case ')': + type = TOKEN_RIGHT_PAREN; + break; + case '[': + type = TOKEN_LEFT_BRACKET; + break; + case ']': + type = TOKEN_RIGHT_BRACKET; + break; + case '{': + type = TOKEN_LEFT_CURLY; + break; + case '}': + type = TOKEN_RIGHT_CURLY; + break; + case ',': + type = TOKEN_COMMA; + break; + case ':': + type = TOKEN_COLON; + break; + case '~': + type = TOKEN_TILDE; + break; + case '|': + type = TOKEN_PIPE; + break; + default: + type = TOKEN_OPERATOR; + } + return token(type, cur, lineno, colno); + } else { + // We are not at whitespace or a delimiter, so extract the + // text and parse it + tok = this._extractUntil(whitespaceChars + delimChars); + if (tok.match(/^[-+]?[0-9]+$/)) { + if (this.current() === '.') { + this.forward(); + var dec = this._extract(intChars); + return token(TOKEN_FLOAT, tok + '.' + dec, lineno, colno); + } else { + return token(TOKEN_INT, tok, lineno, colno); + } + } else if (tok.match(/^(true|false)$/)) { + return token(TOKEN_BOOLEAN, tok, lineno, colno); + } else if (tok === 'none') { + return token(TOKEN_NONE, tok, lineno, colno); + /* + * Added to make the test `null is null` evaluate truthily. + * Otherwise, Nunjucks will look up null in the context and + * return `undefined`, which is not what we want. This *may* have + * consequences is someone is using null in their templates as a + * variable. + */ + } else if (tok === 'null') { + return token(TOKEN_NONE, tok, lineno, colno); + } else if (tok) { + return token(TOKEN_SYMBOL, tok, lineno, colno); + } else { + throw new Error('Unexpected value while parsing: ' + tok); + } + } + } else { + // Parse out the template text, breaking on tag + // delimiters because we need to look for block/variable start + // tags (don't use the full delimChars for optimization) + var beginChars = this.tags.BLOCK_START.charAt(0) + this.tags.VARIABLE_START.charAt(0) + this.tags.COMMENT_START.charAt(0) + this.tags.COMMENT_END.charAt(0); + if (this.isFinished()) { + return null; + } else if ((tok = this._extractString(this.tags.BLOCK_START + '-')) || (tok = this._extractString(this.tags.BLOCK_START))) { + this.in_code = true; + return token(TOKEN_BLOCK_START, tok, lineno, colno); + } else if ((tok = this._extractString(this.tags.VARIABLE_START + '-')) || (tok = this._extractString(this.tags.VARIABLE_START))) { + this.in_code = true; + return token(TOKEN_VARIABLE_START, tok, lineno, colno); + } else { + tok = ''; + var data; + var inComment = false; + if (this._matches(this.tags.COMMENT_START)) { + inComment = true; + tok = this._extractString(this.tags.COMMENT_START); + } + + // Continually consume text, breaking on the tag delimiter + // characters and checking to see if it's a start tag. + // + // We could hit the end of the template in the middle of + // our looping, so check for the null return value from + // _extractUntil + while ((data = this._extractUntil(beginChars)) !== null) { + tok += data; + if ((this._matches(this.tags.BLOCK_START) || this._matches(this.tags.VARIABLE_START) || this._matches(this.tags.COMMENT_START)) && !inComment) { + if (this.lstripBlocks && this._matches(this.tags.BLOCK_START) && this.colno > 0 && this.colno <= tok.length) { + var lastLine = tok.slice(-this.colno); + if (/^\s+$/.test(lastLine)) { + // Remove block leading whitespace from beginning of the string + tok = tok.slice(0, -this.colno); + if (!tok.length) { + // All data removed, collapse to avoid unnecessary nodes + // by returning next token (block start) + return this.nextToken(); + } + } + } + // If it is a start tag, stop looping + break; + } else if (this._matches(this.tags.COMMENT_END)) { + if (!inComment) { + throw new Error('unexpected end of comment'); + } + tok += this._extractString(this.tags.COMMENT_END); + break; + } else { + // It does not match any tag, so add the character and + // carry on + tok += this.current(); + this.forward(); + } + } + if (data === null && inComment) { + throw new Error('expected end of comment, got end of file'); + } + return token(inComment ? TOKEN_COMMENT : TOKEN_DATA, tok, lineno, colno); + } + } + }; + _proto._parseString = function _parseString(delimiter) { + this.forward(); + var str = ''; + while (!this.isFinished() && this.current() !== delimiter) { + var cur = this.current(); + if (cur === '\\') { + this.forward(); + switch (this.current()) { + case 'n': + str += '\n'; + break; + case 't': + str += '\t'; + break; + case 'r': + str += '\r'; + break; + default: + str += this.current(); + } + this.forward(); + } else { + str += cur; + this.forward(); + } + } + this.forward(); + return str; + }; + _proto._matches = function _matches(str) { + if (this.index + str.length > this.len) { + return null; + } + var m = this.str.slice(this.index, this.index + str.length); + return m === str; + }; + _proto._extractString = function _extractString(str) { + if (this._matches(str)) { + this.forwardN(str.length); + return str; + } + return null; + }; + _proto._extractUntil = function _extractUntil(charString) { + // Extract all non-matching chars, with the default matching set + // to everything + return this._extractMatching(true, charString || ''); + }; + _proto._extract = function _extract(charString) { + // Extract all matching chars (no default, so charString must be + // explicit) + return this._extractMatching(false, charString); + }; + _proto._extractMatching = function _extractMatching(breakOnMatch, charString) { + // Pull out characters until a breaking char is hit. + // If breakOnMatch is false, a non-matching char stops it. + // If breakOnMatch is true, a matching char stops it. + + if (this.isFinished()) { + return null; + } + var first = charString.indexOf(this.current()); + + // Only proceed if the first character doesn't meet our condition + if (breakOnMatch && first === -1 || !breakOnMatch && first !== -1) { + var t = this.current(); + this.forward(); + + // And pull out all the chars one at a time until we hit a + // breaking char + var idx = charString.indexOf(this.current()); + while ((breakOnMatch && idx === -1 || !breakOnMatch && idx !== -1) && !this.isFinished()) { + t += this.current(); + this.forward(); + idx = charString.indexOf(this.current()); + } + return t; + } + return ''; + }; + _proto._extractRegex = function _extractRegex(regex) { + var matches = this.currentStr().match(regex); + if (!matches) { + return null; + } + + // Move forward whatever was matched + this.forwardN(matches[0].length); + return matches; + }; + _proto.isFinished = function isFinished() { + return this.index >= this.len; + }; + _proto.forwardN = function forwardN(n) { + for (var i = 0; i < n; i++) { + this.forward(); + } + }; + _proto.forward = function forward() { + this.index++; + if (this.previous() === '\n') { + this.lineno++; + this.colno = 0; + } else { + this.colno++; + } + }; + _proto.backN = function backN(n) { + for (var i = 0; i < n; i++) { + this.back(); + } + }; + _proto.back = function back() { + this.index--; + if (this.current() === '\n') { + this.lineno--; + var idx = this.src.lastIndexOf('\n', this.index - 1); + if (idx === -1) { + this.colno = this.index; + } else { + this.colno = this.index - idx; + } + } else { + this.colno--; + } + } + + // current returns current character + ; + _proto.current = function current() { + if (!this.isFinished()) { + return this.str.charAt(this.index); + } + return ''; + } + + // currentStr returns what's left of the unparsed string + ; + _proto.currentStr = function currentStr() { + if (!this.isFinished()) { + return this.str.substr(this.index); + } + return ''; + }; + _proto.previous = function previous() { + return this.str.charAt(this.index - 1); + }; + return Tokenizer; +}(); +module.exports = { + lex: function lex(src, opts) { + return new Tokenizer(src, opts); + }, + TOKEN_STRING: TOKEN_STRING, + TOKEN_WHITESPACE: TOKEN_WHITESPACE, + TOKEN_DATA: TOKEN_DATA, + TOKEN_BLOCK_START: TOKEN_BLOCK_START, + TOKEN_BLOCK_END: TOKEN_BLOCK_END, + TOKEN_VARIABLE_START: TOKEN_VARIABLE_START, + TOKEN_VARIABLE_END: TOKEN_VARIABLE_END, + TOKEN_COMMENT: TOKEN_COMMENT, + TOKEN_LEFT_PAREN: TOKEN_LEFT_PAREN, + TOKEN_RIGHT_PAREN: TOKEN_RIGHT_PAREN, + TOKEN_LEFT_BRACKET: TOKEN_LEFT_BRACKET, + TOKEN_RIGHT_BRACKET: TOKEN_RIGHT_BRACKET, + TOKEN_LEFT_CURLY: TOKEN_LEFT_CURLY, + TOKEN_RIGHT_CURLY: TOKEN_RIGHT_CURLY, + TOKEN_OPERATOR: TOKEN_OPERATOR, + TOKEN_COMMA: TOKEN_COMMA, + TOKEN_COLON: TOKEN_COLON, + TOKEN_TILDE: TOKEN_TILDE, + TOKEN_PIPE: TOKEN_PIPE, + TOKEN_INT: TOKEN_INT, + TOKEN_FLOAT: TOKEN_FLOAT, + TOKEN_BOOLEAN: TOKEN_BOOLEAN, + TOKEN_NONE: TOKEN_NONE, + TOKEN_SYMBOL: TOKEN_SYMBOL, + TOKEN_SPECIAL: TOKEN_SPECIAL, + TOKEN_REGEX: TOKEN_REGEX +}; \ No newline at end of file -- cgit v1.2.3