summaryrefslogtreecommitdiff
path: root/node_modules/nunjucks/src/lexer.js
diff options
context:
space:
mode:
authorShipwreckt <me@shipwreckt.co.uk>2025-10-31 20:02:14 +0000
committerShipwreckt <me@shipwreckt.co.uk>2025-10-31 20:02:14 +0000
commit7a52ddeba2a68388b544f529d2d92104420f77b0 (patch)
tree15ddd47457a2cb4a96060747437d36474e4f6b4e /node_modules/nunjucks/src/lexer.js
parent53d6ae2b5568437afa5e4995580a3fb679b7b91b (diff)
Changed from static to 11ty!
Diffstat (limited to 'node_modules/nunjucks/src/lexer.js')
-rw-r--r--node_modules/nunjucks/src/lexer.js474
1 files changed, 474 insertions, 0 deletions
diff --git a/node_modules/nunjucks/src/lexer.js b/node_modules/nunjucks/src/lexer.js
new file mode 100644
index 0000000..ca4454d
--- /dev/null
+++ b/node_modules/nunjucks/src/lexer.js
@@ -0,0 +1,474 @@
+'use strict';
+
+var lib = require('./lib');
+var whitespaceChars = " \n\t\r\xA0";
+var delimChars = '()[]{}%*-+~/#,:|.<>=!';
+var intChars = '0123456789';
+var BLOCK_START = '{%';
+var BLOCK_END = '%}';
+var VARIABLE_START = '{{';
+var VARIABLE_END = '}}';
+var COMMENT_START = '{#';
+var COMMENT_END = '#}';
+var TOKEN_STRING = 'string';
+var TOKEN_WHITESPACE = 'whitespace';
+var TOKEN_DATA = 'data';
+var TOKEN_BLOCK_START = 'block-start';
+var TOKEN_BLOCK_END = 'block-end';
+var TOKEN_VARIABLE_START = 'variable-start';
+var TOKEN_VARIABLE_END = 'variable-end';
+var TOKEN_COMMENT = 'comment';
+var TOKEN_LEFT_PAREN = 'left-paren';
+var TOKEN_RIGHT_PAREN = 'right-paren';
+var TOKEN_LEFT_BRACKET = 'left-bracket';
+var TOKEN_RIGHT_BRACKET = 'right-bracket';
+var TOKEN_LEFT_CURLY = 'left-curly';
+var TOKEN_RIGHT_CURLY = 'right-curly';
+var TOKEN_OPERATOR = 'operator';
+var TOKEN_COMMA = 'comma';
+var TOKEN_COLON = 'colon';
+var TOKEN_TILDE = 'tilde';
+var TOKEN_PIPE = 'pipe';
+var TOKEN_INT = 'int';
+var TOKEN_FLOAT = 'float';
+var TOKEN_BOOLEAN = 'boolean';
+var TOKEN_NONE = 'none';
+var TOKEN_SYMBOL = 'symbol';
+var TOKEN_SPECIAL = 'special';
+var TOKEN_REGEX = 'regex';
+function token(type, value, lineno, colno) {
+ return {
+ type: type,
+ value: value,
+ lineno: lineno,
+ colno: colno
+ };
+}
+var Tokenizer = /*#__PURE__*/function () {
+ function Tokenizer(str, opts) {
+ this.str = str;
+ this.index = 0;
+ this.len = str.length;
+ this.lineno = 0;
+ this.colno = 0;
+ this.in_code = false;
+ opts = opts || {};
+ var tags = opts.tags || {};
+ this.tags = {
+ BLOCK_START: tags.blockStart || BLOCK_START,
+ BLOCK_END: tags.blockEnd || BLOCK_END,
+ VARIABLE_START: tags.variableStart || VARIABLE_START,
+ VARIABLE_END: tags.variableEnd || VARIABLE_END,
+ COMMENT_START: tags.commentStart || COMMENT_START,
+ COMMENT_END: tags.commentEnd || COMMENT_END
+ };
+ this.trimBlocks = !!opts.trimBlocks;
+ this.lstripBlocks = !!opts.lstripBlocks;
+ }
+ var _proto = Tokenizer.prototype;
+ _proto.nextToken = function nextToken() {
+ var lineno = this.lineno;
+ var colno = this.colno;
+ var tok;
+ if (this.in_code) {
+ // Otherwise, if we are in a block parse it as code
+ var cur = this.current();
+ if (this.isFinished()) {
+ // We have nothing else to parse
+ return null;
+ } else if (cur === '"' || cur === '\'') {
+ // We've hit a string
+ return token(TOKEN_STRING, this._parseString(cur), lineno, colno);
+ } else if (tok = this._extract(whitespaceChars)) {
+ // We hit some whitespace
+ return token(TOKEN_WHITESPACE, tok, lineno, colno);
+ } else if ((tok = this._extractString(this.tags.BLOCK_END)) || (tok = this._extractString('-' + this.tags.BLOCK_END))) {
+ // Special check for the block end tag
+ //
+ // It is a requirement that start and end tags are composed of
+ // delimiter characters (%{}[] etc), and our code always
+ // breaks on delimiters so we can assume the token parsing
+ // doesn't consume these elsewhere
+ this.in_code = false;
+ if (this.trimBlocks) {
+ cur = this.current();
+ if (cur === '\n') {
+ // Skip newline
+ this.forward();
+ } else if (cur === '\r') {
+ // Skip CRLF newline
+ this.forward();
+ cur = this.current();
+ if (cur === '\n') {
+ this.forward();
+ } else {
+ // Was not a CRLF, so go back
+ this.back();
+ }
+ }
+ }
+ return token(TOKEN_BLOCK_END, tok, lineno, colno);
+ } else if ((tok = this._extractString(this.tags.VARIABLE_END)) || (tok = this._extractString('-' + this.tags.VARIABLE_END))) {
+ // Special check for variable end tag (see above)
+ this.in_code = false;
+ return token(TOKEN_VARIABLE_END, tok, lineno, colno);
+ } else if (cur === 'r' && this.str.charAt(this.index + 1) === '/') {
+ // Skip past 'r/'.
+ this.forwardN(2);
+
+ // Extract until the end of the regex -- / ends it, \/ does not.
+ var regexBody = '';
+ while (!this.isFinished()) {
+ if (this.current() === '/' && this.previous() !== '\\') {
+ this.forward();
+ break;
+ } else {
+ regexBody += this.current();
+ this.forward();
+ }
+ }
+
+ // Check for flags.
+ // The possible flags are according to https://developer.mozilla.org/en/docs/Web/JavaScript/Reference/Global_Objects/RegExp)
+ var POSSIBLE_FLAGS = ['g', 'i', 'm', 'y'];
+ var regexFlags = '';
+ while (!this.isFinished()) {
+ var isCurrentAFlag = POSSIBLE_FLAGS.indexOf(this.current()) !== -1;
+ if (isCurrentAFlag) {
+ regexFlags += this.current();
+ this.forward();
+ } else {
+ break;
+ }
+ }
+ return token(TOKEN_REGEX, {
+ body: regexBody,
+ flags: regexFlags
+ }, lineno, colno);
+ } else if (delimChars.indexOf(cur) !== -1) {
+ // We've hit a delimiter (a special char like a bracket)
+ this.forward();
+ var complexOps = ['==', '===', '!=', '!==', '<=', '>=', '//', '**'];
+ var curComplex = cur + this.current();
+ var type;
+ if (lib.indexOf(complexOps, curComplex) !== -1) {
+ this.forward();
+ cur = curComplex;
+
+ // See if this is a strict equality/inequality comparator
+ if (lib.indexOf(complexOps, curComplex + this.current()) !== -1) {
+ cur = curComplex + this.current();
+ this.forward();
+ }
+ }
+ switch (cur) {
+ case '(':
+ type = TOKEN_LEFT_PAREN;
+ break;
+ case ')':
+ type = TOKEN_RIGHT_PAREN;
+ break;
+ case '[':
+ type = TOKEN_LEFT_BRACKET;
+ break;
+ case ']':
+ type = TOKEN_RIGHT_BRACKET;
+ break;
+ case '{':
+ type = TOKEN_LEFT_CURLY;
+ break;
+ case '}':
+ type = TOKEN_RIGHT_CURLY;
+ break;
+ case ',':
+ type = TOKEN_COMMA;
+ break;
+ case ':':
+ type = TOKEN_COLON;
+ break;
+ case '~':
+ type = TOKEN_TILDE;
+ break;
+ case '|':
+ type = TOKEN_PIPE;
+ break;
+ default:
+ type = TOKEN_OPERATOR;
+ }
+ return token(type, cur, lineno, colno);
+ } else {
+ // We are not at whitespace or a delimiter, so extract the
+ // text and parse it
+ tok = this._extractUntil(whitespaceChars + delimChars);
+ if (tok.match(/^[-+]?[0-9]+$/)) {
+ if (this.current() === '.') {
+ this.forward();
+ var dec = this._extract(intChars);
+ return token(TOKEN_FLOAT, tok + '.' + dec, lineno, colno);
+ } else {
+ return token(TOKEN_INT, tok, lineno, colno);
+ }
+ } else if (tok.match(/^(true|false)$/)) {
+ return token(TOKEN_BOOLEAN, tok, lineno, colno);
+ } else if (tok === 'none') {
+ return token(TOKEN_NONE, tok, lineno, colno);
+ /*
+ * Added to make the test `null is null` evaluate truthily.
+ * Otherwise, Nunjucks will look up null in the context and
+ * return `undefined`, which is not what we want. This *may* have
+ * consequences is someone is using null in their templates as a
+ * variable.
+ */
+ } else if (tok === 'null') {
+ return token(TOKEN_NONE, tok, lineno, colno);
+ } else if (tok) {
+ return token(TOKEN_SYMBOL, tok, lineno, colno);
+ } else {
+ throw new Error('Unexpected value while parsing: ' + tok);
+ }
+ }
+ } else {
+ // Parse out the template text, breaking on tag
+ // delimiters because we need to look for block/variable start
+ // tags (don't use the full delimChars for optimization)
+ var beginChars = this.tags.BLOCK_START.charAt(0) + this.tags.VARIABLE_START.charAt(0) + this.tags.COMMENT_START.charAt(0) + this.tags.COMMENT_END.charAt(0);
+ if (this.isFinished()) {
+ return null;
+ } else if ((tok = this._extractString(this.tags.BLOCK_START + '-')) || (tok = this._extractString(this.tags.BLOCK_START))) {
+ this.in_code = true;
+ return token(TOKEN_BLOCK_START, tok, lineno, colno);
+ } else if ((tok = this._extractString(this.tags.VARIABLE_START + '-')) || (tok = this._extractString(this.tags.VARIABLE_START))) {
+ this.in_code = true;
+ return token(TOKEN_VARIABLE_START, tok, lineno, colno);
+ } else {
+ tok = '';
+ var data;
+ var inComment = false;
+ if (this._matches(this.tags.COMMENT_START)) {
+ inComment = true;
+ tok = this._extractString(this.tags.COMMENT_START);
+ }
+
+ // Continually consume text, breaking on the tag delimiter
+ // characters and checking to see if it's a start tag.
+ //
+ // We could hit the end of the template in the middle of
+ // our looping, so check for the null return value from
+ // _extractUntil
+ while ((data = this._extractUntil(beginChars)) !== null) {
+ tok += data;
+ if ((this._matches(this.tags.BLOCK_START) || this._matches(this.tags.VARIABLE_START) || this._matches(this.tags.COMMENT_START)) && !inComment) {
+ if (this.lstripBlocks && this._matches(this.tags.BLOCK_START) && this.colno > 0 && this.colno <= tok.length) {
+ var lastLine = tok.slice(-this.colno);
+ if (/^\s+$/.test(lastLine)) {
+ // Remove block leading whitespace from beginning of the string
+ tok = tok.slice(0, -this.colno);
+ if (!tok.length) {
+ // All data removed, collapse to avoid unnecessary nodes
+ // by returning next token (block start)
+ return this.nextToken();
+ }
+ }
+ }
+ // If it is a start tag, stop looping
+ break;
+ } else if (this._matches(this.tags.COMMENT_END)) {
+ if (!inComment) {
+ throw new Error('unexpected end of comment');
+ }
+ tok += this._extractString(this.tags.COMMENT_END);
+ break;
+ } else {
+ // It does not match any tag, so add the character and
+ // carry on
+ tok += this.current();
+ this.forward();
+ }
+ }
+ if (data === null && inComment) {
+ throw new Error('expected end of comment, got end of file');
+ }
+ return token(inComment ? TOKEN_COMMENT : TOKEN_DATA, tok, lineno, colno);
+ }
+ }
+ };
+ _proto._parseString = function _parseString(delimiter) {
+ this.forward();
+ var str = '';
+ while (!this.isFinished() && this.current() !== delimiter) {
+ var cur = this.current();
+ if (cur === '\\') {
+ this.forward();
+ switch (this.current()) {
+ case 'n':
+ str += '\n';
+ break;
+ case 't':
+ str += '\t';
+ break;
+ case 'r':
+ str += '\r';
+ break;
+ default:
+ str += this.current();
+ }
+ this.forward();
+ } else {
+ str += cur;
+ this.forward();
+ }
+ }
+ this.forward();
+ return str;
+ };
+ _proto._matches = function _matches(str) {
+ if (this.index + str.length > this.len) {
+ return null;
+ }
+ var m = this.str.slice(this.index, this.index + str.length);
+ return m === str;
+ };
+ _proto._extractString = function _extractString(str) {
+ if (this._matches(str)) {
+ this.forwardN(str.length);
+ return str;
+ }
+ return null;
+ };
+ _proto._extractUntil = function _extractUntil(charString) {
+ // Extract all non-matching chars, with the default matching set
+ // to everything
+ return this._extractMatching(true, charString || '');
+ };
+ _proto._extract = function _extract(charString) {
+ // Extract all matching chars (no default, so charString must be
+ // explicit)
+ return this._extractMatching(false, charString);
+ };
+ _proto._extractMatching = function _extractMatching(breakOnMatch, charString) {
+ // Pull out characters until a breaking char is hit.
+ // If breakOnMatch is false, a non-matching char stops it.
+ // If breakOnMatch is true, a matching char stops it.
+
+ if (this.isFinished()) {
+ return null;
+ }
+ var first = charString.indexOf(this.current());
+
+ // Only proceed if the first character doesn't meet our condition
+ if (breakOnMatch && first === -1 || !breakOnMatch && first !== -1) {
+ var t = this.current();
+ this.forward();
+
+ // And pull out all the chars one at a time until we hit a
+ // breaking char
+ var idx = charString.indexOf(this.current());
+ while ((breakOnMatch && idx === -1 || !breakOnMatch && idx !== -1) && !this.isFinished()) {
+ t += this.current();
+ this.forward();
+ idx = charString.indexOf(this.current());
+ }
+ return t;
+ }
+ return '';
+ };
+ _proto._extractRegex = function _extractRegex(regex) {
+ var matches = this.currentStr().match(regex);
+ if (!matches) {
+ return null;
+ }
+
+ // Move forward whatever was matched
+ this.forwardN(matches[0].length);
+ return matches;
+ };
+ _proto.isFinished = function isFinished() {
+ return this.index >= this.len;
+ };
+ _proto.forwardN = function forwardN(n) {
+ for (var i = 0; i < n; i++) {
+ this.forward();
+ }
+ };
+ _proto.forward = function forward() {
+ this.index++;
+ if (this.previous() === '\n') {
+ this.lineno++;
+ this.colno = 0;
+ } else {
+ this.colno++;
+ }
+ };
+ _proto.backN = function backN(n) {
+ for (var i = 0; i < n; i++) {
+ this.back();
+ }
+ };
+ _proto.back = function back() {
+ this.index--;
+ if (this.current() === '\n') {
+ this.lineno--;
+ var idx = this.src.lastIndexOf('\n', this.index - 1);
+ if (idx === -1) {
+ this.colno = this.index;
+ } else {
+ this.colno = this.index - idx;
+ }
+ } else {
+ this.colno--;
+ }
+ }
+
+ // current returns current character
+ ;
+ _proto.current = function current() {
+ if (!this.isFinished()) {
+ return this.str.charAt(this.index);
+ }
+ return '';
+ }
+
+ // currentStr returns what's left of the unparsed string
+ ;
+ _proto.currentStr = function currentStr() {
+ if (!this.isFinished()) {
+ return this.str.substr(this.index);
+ }
+ return '';
+ };
+ _proto.previous = function previous() {
+ return this.str.charAt(this.index - 1);
+ };
+ return Tokenizer;
+}();
+module.exports = {
+ lex: function lex(src, opts) {
+ return new Tokenizer(src, opts);
+ },
+ TOKEN_STRING: TOKEN_STRING,
+ TOKEN_WHITESPACE: TOKEN_WHITESPACE,
+ TOKEN_DATA: TOKEN_DATA,
+ TOKEN_BLOCK_START: TOKEN_BLOCK_START,
+ TOKEN_BLOCK_END: TOKEN_BLOCK_END,
+ TOKEN_VARIABLE_START: TOKEN_VARIABLE_START,
+ TOKEN_VARIABLE_END: TOKEN_VARIABLE_END,
+ TOKEN_COMMENT: TOKEN_COMMENT,
+ TOKEN_LEFT_PAREN: TOKEN_LEFT_PAREN,
+ TOKEN_RIGHT_PAREN: TOKEN_RIGHT_PAREN,
+ TOKEN_LEFT_BRACKET: TOKEN_LEFT_BRACKET,
+ TOKEN_RIGHT_BRACKET: TOKEN_RIGHT_BRACKET,
+ TOKEN_LEFT_CURLY: TOKEN_LEFT_CURLY,
+ TOKEN_RIGHT_CURLY: TOKEN_RIGHT_CURLY,
+ TOKEN_OPERATOR: TOKEN_OPERATOR,
+ TOKEN_COMMA: TOKEN_COMMA,
+ TOKEN_COLON: TOKEN_COLON,
+ TOKEN_TILDE: TOKEN_TILDE,
+ TOKEN_PIPE: TOKEN_PIPE,
+ TOKEN_INT: TOKEN_INT,
+ TOKEN_FLOAT: TOKEN_FLOAT,
+ TOKEN_BOOLEAN: TOKEN_BOOLEAN,
+ TOKEN_NONE: TOKEN_NONE,
+ TOKEN_SYMBOL: TOKEN_SYMBOL,
+ TOKEN_SPECIAL: TOKEN_SPECIAL,
+ TOKEN_REGEX: TOKEN_REGEX
+}; \ No newline at end of file