mirror of
https://github.com/Luzifer/repo-template.git
synced 2024-09-20 09:32:58 +00:00
422 lines
7.9 KiB
Go
422 lines
7.9 KiB
Go
package pongo2
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
"unicode/utf8"
|
|
)
|
|
|
|
const (
|
|
TokenError = iota
|
|
EOF
|
|
|
|
TokenHTML
|
|
|
|
TokenKeyword
|
|
TokenIdentifier
|
|
TokenString
|
|
TokenNumber
|
|
TokenSymbol
|
|
)
|
|
|
|
var (
|
|
tokenSpaceChars = " \n\r\t"
|
|
tokenIdentifierChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_"
|
|
tokenIdentifierCharsWithDigits = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_0123456789"
|
|
tokenDigits = "0123456789"
|
|
|
|
// Available symbols in pongo2 (within filters/tag)
|
|
TokenSymbols = []string{
|
|
// 3-Char symbols
|
|
|
|
// 2-Char symbols
|
|
"==", ">=", "<=", "&&", "||", "{{", "}}", "{%", "%}", "!=", "<>",
|
|
|
|
// 1-Char symbol
|
|
"(", ")", "+", "-", "*", "<", ">", "/", "^", ",", ".", "!", "|", ":", "=", "%",
|
|
}
|
|
|
|
// Available keywords in pongo2
|
|
TokenKeywords = []string{"in", "and", "or", "not", "true", "false", "as", "export"}
|
|
)
|
|
|
|
type TokenType int
|
|
type Token struct {
|
|
Filename string
|
|
Typ TokenType
|
|
Val string
|
|
Line int
|
|
Col int
|
|
}
|
|
|
|
type lexerStateFn func() lexerStateFn
|
|
type lexer struct {
|
|
name string
|
|
input string
|
|
start int // start pos of the item
|
|
pos int // current pos
|
|
width int // width of last rune
|
|
tokens []*Token
|
|
errored bool
|
|
startline int
|
|
startcol int
|
|
line int
|
|
col int
|
|
|
|
in_verbatim bool
|
|
verbatim_name string
|
|
}
|
|
|
|
func (t *Token) String() string {
|
|
val := t.Val
|
|
if len(val) > 1000 {
|
|
val = fmt.Sprintf("%s...%s", val[:10], val[len(val)-5:len(val)])
|
|
}
|
|
|
|
typ := ""
|
|
switch t.Typ {
|
|
case TokenHTML:
|
|
typ = "HTML"
|
|
case TokenError:
|
|
typ = "Error"
|
|
case TokenIdentifier:
|
|
typ = "Identifier"
|
|
case TokenKeyword:
|
|
typ = "Keyword"
|
|
case TokenNumber:
|
|
typ = "Number"
|
|
case TokenString:
|
|
typ = "String"
|
|
case TokenSymbol:
|
|
typ = "Symbol"
|
|
default:
|
|
typ = "Unknown"
|
|
}
|
|
|
|
return fmt.Sprintf("<Token Typ=%s (%d) Val='%s' Line=%d Col=%d>",
|
|
typ, t.Typ, val, t.Line, t.Col)
|
|
}
|
|
|
|
func lex(name string, input string) ([]*Token, *Error) {
|
|
l := &lexer{
|
|
name: name,
|
|
input: input,
|
|
tokens: make([]*Token, 0, 100),
|
|
line: 1,
|
|
col: 1,
|
|
startline: 1,
|
|
startcol: 1,
|
|
}
|
|
l.run()
|
|
if l.errored {
|
|
errtoken := l.tokens[len(l.tokens)-1]
|
|
return nil, &Error{
|
|
Filename: name,
|
|
Line: errtoken.Line,
|
|
Column: errtoken.Col,
|
|
Sender: "lexer",
|
|
ErrorMsg: errtoken.Val,
|
|
}
|
|
}
|
|
return l.tokens, nil
|
|
}
|
|
|
|
func (l *lexer) value() string {
|
|
return l.input[l.start:l.pos]
|
|
}
|
|
|
|
func (l *lexer) length() int {
|
|
return l.pos - l.start
|
|
}
|
|
|
|
func (l *lexer) emit(t TokenType) {
|
|
tok := &Token{
|
|
Filename: l.name,
|
|
Typ: t,
|
|
Val: l.value(),
|
|
Line: l.startline,
|
|
Col: l.startcol,
|
|
}
|
|
|
|
if t == TokenString {
|
|
// Escape sequence \" in strings
|
|
tok.Val = strings.Replace(tok.Val, `\"`, `"`, -1)
|
|
tok.Val = strings.Replace(tok.Val, `\\`, `\`, -1)
|
|
}
|
|
|
|
l.tokens = append(l.tokens, tok)
|
|
l.start = l.pos
|
|
l.startline = l.line
|
|
l.startcol = l.col
|
|
}
|
|
|
|
func (l *lexer) next() rune {
|
|
if l.pos >= len(l.input) {
|
|
l.width = 0
|
|
return EOF
|
|
}
|
|
r, w := utf8.DecodeRuneInString(l.input[l.pos:])
|
|
l.width = w
|
|
l.pos += l.width
|
|
l.col += l.width
|
|
return r
|
|
}
|
|
|
|
func (l *lexer) backup() {
|
|
l.pos -= l.width
|
|
l.col -= l.width
|
|
}
|
|
|
|
func (l *lexer) peek() rune {
|
|
r := l.next()
|
|
l.backup()
|
|
return r
|
|
}
|
|
|
|
func (l *lexer) ignore() {
|
|
l.start = l.pos
|
|
l.startline = l.line
|
|
l.startcol = l.col
|
|
}
|
|
|
|
func (l *lexer) accept(what string) bool {
|
|
if strings.IndexRune(what, l.next()) >= 0 {
|
|
return true
|
|
}
|
|
l.backup()
|
|
return false
|
|
}
|
|
|
|
func (l *lexer) acceptRun(what string) {
|
|
for strings.IndexRune(what, l.next()) >= 0 {
|
|
}
|
|
l.backup()
|
|
}
|
|
|
|
func (l *lexer) errorf(format string, args ...interface{}) lexerStateFn {
|
|
t := &Token{
|
|
Filename: l.name,
|
|
Typ: TokenError,
|
|
Val: fmt.Sprintf(format, args...),
|
|
Line: l.startline,
|
|
Col: l.startcol,
|
|
}
|
|
l.tokens = append(l.tokens, t)
|
|
l.errored = true
|
|
l.startline = l.line
|
|
l.startcol = l.col
|
|
return nil
|
|
}
|
|
|
|
func (l *lexer) eof() bool {
|
|
return l.start >= len(l.input)-1
|
|
}
|
|
|
|
func (l *lexer) run() {
|
|
for {
|
|
// TODO: Support verbatim tag names
|
|
// https://docs.djangoproject.com/en/dev/ref/templates/builtins/#verbatim
|
|
if l.in_verbatim {
|
|
name := l.verbatim_name
|
|
if name != "" {
|
|
name += " "
|
|
}
|
|
if strings.HasPrefix(l.input[l.pos:], fmt.Sprintf("{%% endverbatim %s%%}", name)) { // end verbatim
|
|
if l.pos > l.start {
|
|
l.emit(TokenHTML)
|
|
}
|
|
w := len("{% endverbatim %}")
|
|
l.pos += w
|
|
l.col += w
|
|
l.ignore()
|
|
l.in_verbatim = false
|
|
}
|
|
} else if strings.HasPrefix(l.input[l.pos:], "{% verbatim %}") { // tag
|
|
if l.pos > l.start {
|
|
l.emit(TokenHTML)
|
|
}
|
|
l.in_verbatim = true
|
|
w := len("{% verbatim %}")
|
|
l.pos += w
|
|
l.col += w
|
|
l.ignore()
|
|
}
|
|
|
|
if !l.in_verbatim {
|
|
// Ignore single-line comments {# ... #}
|
|
if strings.HasPrefix(l.input[l.pos:], "{#") {
|
|
if l.pos > l.start {
|
|
l.emit(TokenHTML)
|
|
}
|
|
|
|
l.pos += 2 // pass '{#'
|
|
l.col += 2
|
|
|
|
for {
|
|
switch l.peek() {
|
|
case EOF:
|
|
l.errorf("Single-line comment not closed.")
|
|
return
|
|
case '\n':
|
|
l.errorf("Newline not permitted in a single-line comment.")
|
|
return
|
|
}
|
|
|
|
if strings.HasPrefix(l.input[l.pos:], "#}") {
|
|
l.pos += 2 // pass '#}'
|
|
l.col += 2
|
|
break
|
|
}
|
|
|
|
l.next()
|
|
}
|
|
l.ignore() // ignore whole comment
|
|
|
|
// Comment skipped
|
|
continue // next token
|
|
}
|
|
|
|
if strings.HasPrefix(l.input[l.pos:], "{{") || // variable
|
|
strings.HasPrefix(l.input[l.pos:], "{%") { // tag
|
|
if l.pos > l.start {
|
|
l.emit(TokenHTML)
|
|
}
|
|
l.tokenize()
|
|
if l.errored {
|
|
return
|
|
}
|
|
continue
|
|
}
|
|
}
|
|
|
|
switch l.peek() {
|
|
case '\n':
|
|
l.line++
|
|
l.col = 0
|
|
}
|
|
if l.next() == EOF {
|
|
break
|
|
}
|
|
}
|
|
|
|
if l.pos > l.start {
|
|
l.emit(TokenHTML)
|
|
}
|
|
|
|
if l.in_verbatim {
|
|
l.errorf("verbatim-tag not closed, got EOF.")
|
|
}
|
|
}
|
|
|
|
func (l *lexer) tokenize() {
|
|
for state := l.stateCode; state != nil; {
|
|
state = state()
|
|
}
|
|
}
|
|
|
|
func (l *lexer) stateCode() lexerStateFn {
|
|
outer_loop:
|
|
for {
|
|
switch {
|
|
case l.accept(tokenSpaceChars):
|
|
if l.value() == "\n" {
|
|
return l.errorf("Newline not allowed within tag/variable.")
|
|
}
|
|
l.ignore()
|
|
continue
|
|
case l.accept(tokenIdentifierChars):
|
|
return l.stateIdentifier
|
|
case l.accept(tokenDigits):
|
|
return l.stateNumber
|
|
case l.accept(`"`):
|
|
return l.stateString
|
|
}
|
|
|
|
// Check for symbol
|
|
for _, sym := range TokenSymbols {
|
|
if strings.HasPrefix(l.input[l.start:], sym) {
|
|
l.pos += len(sym)
|
|
l.col += l.length()
|
|
l.emit(TokenSymbol)
|
|
|
|
if sym == "%}" || sym == "}}" {
|
|
// Tag/variable end, return after emit
|
|
return nil
|
|
}
|
|
|
|
continue outer_loop
|
|
}
|
|
}
|
|
|
|
if l.pos < len(l.input) {
|
|
return l.errorf("Unknown character: %q (%d)", l.peek(), l.peek())
|
|
}
|
|
|
|
break
|
|
}
|
|
|
|
// Normal shut down
|
|
return nil
|
|
}
|
|
|
|
func (l *lexer) stateIdentifier() lexerStateFn {
|
|
l.acceptRun(tokenIdentifierChars)
|
|
l.acceptRun(tokenIdentifierCharsWithDigits)
|
|
for _, kw := range TokenKeywords {
|
|
if kw == l.value() {
|
|
l.emit(TokenKeyword)
|
|
return l.stateCode
|
|
}
|
|
}
|
|
l.emit(TokenIdentifier)
|
|
return l.stateCode
|
|
}
|
|
|
|
func (l *lexer) stateNumber() lexerStateFn {
|
|
l.acceptRun(tokenDigits)
|
|
/*
|
|
Maybe context-sensitive number lexing?
|
|
* comments.0.Text // first comment
|
|
* usercomments.1.0 // second user, first comment
|
|
* if (score >= 8.5) // 8.5 as a number
|
|
|
|
if l.peek() == '.' {
|
|
l.accept(".")
|
|
if !l.accept(tokenDigits) {
|
|
return l.errorf("Malformed number.")
|
|
}
|
|
l.acceptRun(tokenDigits)
|
|
}
|
|
*/
|
|
l.emit(TokenNumber)
|
|
return l.stateCode
|
|
}
|
|
|
|
func (l *lexer) stateString() lexerStateFn {
|
|
l.ignore()
|
|
l.startcol -= 1 // we're starting the position at the first "
|
|
for !l.accept(`"`) {
|
|
switch l.next() {
|
|
case '\\':
|
|
// escape sequence
|
|
switch l.peek() {
|
|
case '"', '\\':
|
|
l.next()
|
|
default:
|
|
return l.errorf("Unknown escape sequence: \\%c", l.peek())
|
|
}
|
|
case EOF:
|
|
return l.errorf("Unexpected EOF, string not closed.")
|
|
case '\n':
|
|
return l.errorf("Newline in string is not allowed.")
|
|
}
|
|
}
|
|
l.backup()
|
|
l.emit(TokenString)
|
|
|
|
l.next()
|
|
l.ignore()
|
|
|
|
return l.stateCode
|
|
}
|