package toml import ( "cmp" "errors" "fmt" "reflect" "slices" "strconv" "unicode/utf8" ) // Parser parses TOML tokens into a map[string]any type Parser struct { root map[string]any lexer Lexer current map[string]any // the table keys are assigned to // Inline tables a key's value made, immutable per TOML spec: a few in place, // a set past them. One in an array no key can reach, so it is not held. frozen [4]uintptr nfrozen int frozenSet map[uintptr]bool curToken Token peekToken Token // Value nesting depth (arrays / inline tables) depth int // Tables (maps) a document may create; 0 means DefaultMaxTables MaxTables int tables int // Elements of the arrays being parsed, an inner array's above its parent's, // so each array is one allocation of its final length values []any } // Recursion bound for parseValue -> parseArray/parseInlineTable const maxValueDepth = 1000 // DefaultMaxTables bounds the maps one document creates. Every dotted-key // segment and header part is a map of about 350 bytes, so without a bound a // 10 MiB file of dotted keys holds near 2 GB. const DefaultMaxTables = 1 << 16 func NewParser(input []byte) *Parser { p := &Parser{lexer: Lexer{input: input, line: 1, lean: true}, root: make(map[string]any)} p.current = p.root p.nextToken() p.nextToken() return p } // newTable makes a map, counted against the document's table budget func (p *Parser) newTable() (map[string]any, error) { p.tables++ if limit := cmp.Or(p.MaxTables, DefaultMaxTables); p.tables > limit { return nil, fmt.Errorf("document exceeds %d tables at line %d", limit, p.curToken.Line) } return make(map[string]any), nil } func (p *Parser) nextToken() { p.curToken = p.peekToken p.peekToken = p.lexer.NextToken() // Skip comments automatically for p.peekToken.Type == TokenComment { p.peekToken = p.lexer.NextToken() } } func (p *Parser) Parse() (map[string]any, error) { if !utf8.Valid(p.lexer.input) { return nil, fmt.Errorf("invalid UTF-8") } for p.curToken.Type != TokenEOF { if p.curToken.Type == TokenNewline { p.nextToken() continue } if err := p.parseStatement(); err != nil { return nil, err } if p.curToken.Type != TokenNewline && p.curToken.Type != TokenEOF { return nil, fmt.Errorf("expected newline at line %d", p.curToken.Line) } } return p.root, nil } func (p *Parser) parseStatement() error { switch p.curToken.Type { case TokenLBracket: // Table Definition: [table] or [[array.table]] return p.parseTableDeclaration() case TokenIdent, TokenString: // Key-Value Pair: key = value return p.parseKeyValuePair(p.current) case TokenError: return fmt.Errorf("lexing error line %d: %s", p.curToken.Line, p.curToken.Literal) default: return fmt.Errorf("unexpected token line %d: %s", p.curToken.Line, p.show(p.curToken)) } } // parseTableDeclaration handles [key] and [[key]]; the doubled brackets of // an array of tables are one delimiter, so "[ [a] ]" is no header func (p *Parser) parseTableDeclaration() error { isArray := false if p.peekToken.Type == TokenLBracket && adjacent(p.curToken, p.peekToken) { // It is [[ ... p.nextToken() // consume first [ isArray = true } p.nextToken() // consume [ // Parse Key (dotted) var buf [4]string keys, err := p.parseKeyParts(buf[:0]) if err != nil { return err } if isArray { if p.curToken.Type != TokenRBracket || p.peekToken.Type != TokenRBracket || !adjacent(p.curToken, p.peekToken) { return fmt.Errorf("expected ]] closing array table at line %d", p.curToken.Line) } p.nextToken() // consume first ] } if p.curToken.Type != TokenRBracket { return fmt.Errorf("expected closing bracket for table at line %d", p.curToken.Line) } p.nextToken() // consume final ] // Define scope return p.setTableScope(keys, isArray) } // setTableScope navigates/creates the map structure and sets p.current func (p *Parser) setTableScope(keys []string, isArrayOfTables bool) error { // Table declarations always start from root var ptr any = p.root for i, key := range keys { isLast := i == len(keys)-1 currentMap, ok := ptr.(map[string]any) if !ok { return fmt.Errorf("key path conflict: %q is not a map", key) } if isLast { if isArrayOfTables { // [[a.b]] -> Ensure 'b' is a slice of maps, append new map, set cursor to it var slice []map[string]any if val, exists := currentMap[key]; exists { if s, ok := val.([]map[string]any); ok { slice = s } else { return fmt.Errorf("key conflict: %q is not an array of tables", key) } } else { slice = make([]map[string]any, 0) } newMap, err := p.newTable() if err != nil { return err } slice = append(slice, newMap) currentMap[key] = slice p.current = newMap } else { // [a.b] -> Ensure 'b' is a map, set cursor to it var targetMap map[string]any if val, exists := currentMap[key]; exists { if m, ok := val.(map[string]any); ok { // Inline tables cannot be reopened if p.isFrozen(m) { return fmt.Errorf("cannot extend inline table %q at line %d", key, p.curToken.Line) } targetMap = m } else { return fmt.Errorf("key conflict: %q is not a table", key) } } else { var err error if targetMap, err = p.newTable(); err != nil { return err } currentMap[key] = targetMap } p.current = targetMap } } else { // Intermediate key -> ensure map exists and traverse. // Traversal through an existing [[array]] descends into its last element. if val, exists := currentMap[key]; exists { if m, ok := val.(map[string]any); ok { // Inline tables cannot be extended via sub-tables if p.isFrozen(m) { return fmt.Errorf("cannot extend inline table %q at line %d", key, p.curToken.Line) } ptr = m } else if slice, ok := val.([]map[string]any); ok { if len(slice) == 0 { return fmt.Errorf("cannot traverse empty array table %q", key) } ptr = slice[len(slice)-1] } else { return fmt.Errorf("intermediate key %q is not a map", key) } } else { newMap, err := p.newTable() if err != nil { return err } currentMap[key] = newMap ptr = newMap } } } return nil } func (p *Parser) parseKeyValuePair(scope map[string]any) error { // Parse Key (dotted allowed: a.b.c = 1) var buf [4]string keys, err := p.parseKeyParts(buf[:0]) if err != nil { return err } if p.curToken.Type != TokenEqual { return fmt.Errorf("expected '=' after key at line %d, got %s", p.curToken.Line, p.show(p.curToken)) } p.nextToken() // consume = val, err := p.parseValue() if err != nil { return err } return p.assignValue(scope, keys, val) } // assignValue sets keys under scope to val, freezing val if it is an inline table func (p *Parser) assignValue(scope map[string]any, keys []string, val any) error { currentMap := scope for i, key := range keys { if i == len(keys)-1 { // Final key, assign value if _, exists := currentMap[key]; exists { return fmt.Errorf("duplicate key %q at line %d", key, p.curToken.Line) } currentMap[key] = val if m, ok := val.(map[string]any); ok { p.freeze(m) } } else { // Intermediate, ensure map if existing, exists := currentMap[key]; exists { if m, ok := existing.(map[string]any); ok { if p.isFrozen(m) { return fmt.Errorf("cannot extend inline table %q at line %d", key, p.curToken.Line) } currentMap = m } else { return fmt.Errorf("intermediate key %q is not a map", key) } } else { newMap, err := p.newTable() if err != nil { return err } currentMap[key] = newMap currentMap = newMap } } } return nil } // freeze marks an inline table no later key may extend func (p *Parser) freeze(m map[string]any) { k := reflect.ValueOf(m).Pointer() if p.frozenSet == nil && p.nfrozen < len(p.frozen) { p.frozen[p.nfrozen] = k p.nfrozen++ return } if p.frozenSet == nil { p.frozenSet = make(map[uintptr]bool) for _, f := range p.frozen { p.frozenSet[f] = true } } p.frozenSet[k] = true } func (p *Parser) isFrozen(m map[string]any) bool { k := reflect.ValueOf(m).Pointer() if p.frozenSet != nil { return p.frozenSet[k] } return slices.Contains(p.frozen[:p.nfrozen], k) } // parseKeyParts appends a dotted key's parts to keys, which callers keep on // their stack func (p *Parser) parseKeyParts(keys []string) ([]string, error) { for { // Rule: Tokens identified as Numbers are forbidden as keys if p.curToken.Type == TokenInteger || p.curToken.Type == TokenFloat { return nil, fmt.Errorf("numeric keys are forbidden: %s", brief(string(p.lexer.input[p.curToken.Pos:p.curToken.End]))) } if p.curToken.Type == TokenString { // Rule: Even quoted strings shouldn't be pure numbers per instruction if numericKey(p.curToken.Literal) { return nil, fmt.Errorf("numeric string keys are forbidden: %s", brief(p.curToken.Literal)) } } if p.curToken.Type != TokenIdent && p.curToken.Type != TokenString { return nil, fmt.Errorf("expected key, got %s", p.show(p.curToken)) } if len(keys) >= maxValueDepth { return nil, fmt.Errorf("key nesting exceeds %d", maxValueDepth) } keys = append(keys, p.curToken.Literal) p.nextToken() if p.curToken.Type == TokenDot { p.nextToken() continue } break } return keys, nil } func (p *Parser) parseValue() (any, error) { // Guard unbounded recursion on nested p.depth++ defer func() { p.depth-- }() if p.depth > maxValueDepth { return nil, fmt.Errorf("value nesting exceeds %d at line %d", maxValueDepth, p.curToken.Line) } switch p.curToken.Type { case TokenString: val := p.curToken.Literal p.nextToken() return val, nil case TokenInteger: lit := p.lexer.input[p.curToken.Pos:p.curToken.End] val, err := parseInteger(lit) if err != nil { return nil, fmt.Errorf("invalid integer %s at line %d: %w", brief(string(lit)), p.curToken.Line, numCause(err)) } p.nextToken() return val, nil case TokenFloat: lit := p.lexer.input[p.curToken.Pos:p.curToken.End] val, err := strconv.ParseFloat(string(lit), 64) if err != nil { return nil, fmt.Errorf("invalid float %s at line %d: %w", brief(string(lit)), p.curToken.Line, numCause(err)) } p.nextToken() return val, nil case TokenBool: val := p.curToken.Literal == "true" p.nextToken() return val, nil case TokenLBracket: return p.parseArray() case TokenLBrace: return p.parseInlineTable() } if p.curToken.Type == TokenIdent { // An unquoted value may be a secret: say what it is, not what it says return nil, fmt.Errorf("unquoted value at line %d: strings need quotes", p.curToken.Line) } return nil, fmt.Errorf("unexpected value token %s at line %d", p.show(p.curToken), p.curToken.Line) } // parseInteger reads a lexed integer: a prefixed one as strconv's base 0 does, the // rest in base 10, so a leading zero is not octal. Neither keeps lit. func parseInteger(lit []byte) (int64, error) { digits := lit if len(digits) > 0 && (digits[0] == '+' || digits[0] == '-') { digits = digits[1:] } if len(digits) > 2 && digits[0] == '0' && !isDigit(rune(digits[1])) { return strconv.ParseInt(string(lit), 0, 64) } return strconv.ParseInt(string(lit), 10, 64) } func (p *Parser) parseArray() ([]any, error) { p.nextToken() // consume [ base := len(p.values) if p.values == nil { p.values = make([]any, 0, 8) } for p.curToken.Type != TokenRBracket { if p.curToken.Type == TokenNewline { p.nextToken() continue } val, err := p.parseValue() if err != nil { return nil, err } p.values = append(p.values, val) for p.curToken.Type == TokenNewline { p.nextToken() } if p.curToken.Type == TokenComma { p.nextToken() } else if p.curToken.Type != TokenRBracket { return nil, fmt.Errorf("expected comma or closing bracket in array at line %d", p.curToken.Line) } } p.nextToken() // consume ] arr := make([]any, len(p.values)-base) copy(arr, p.values[base:]) clear(p.values[base:]) p.values = p.values[:base] return arr, nil } func (p *Parser) parseInlineTable() (map[string]any, error) { p.nextToken() // consume { m, err := p.newTable() if err != nil { return nil, err } for p.curToken.Type != TokenRBrace { if p.curToken.Type == TokenNewline { p.nextToken() continue } // Parse key = value var buf [4]string keys, err := p.parseKeyParts(buf[:0]) if err != nil { return nil, err } if p.curToken.Type != TokenEqual { return nil, fmt.Errorf("expected '=' in inline table at line %d", p.curToken.Line) } p.nextToken() val, err := p.parseValue() if err != nil { return nil, err } // Inline tables can have dotted keys too: { a.b = 1 } if err := p.assignValue(m, keys, val); err != nil { return nil, err } for p.curToken.Type == TokenNewline { p.nextToken() } if p.curToken.Type == TokenComma { p.nextToken() } else if p.curToken.Type != TokenRBrace { return nil, fmt.Errorf("expected comma or closing brace in inline table at line %d", p.curToken.Line) } } p.nextToken() // consume } // The key this is the value of freezes it (assignValue), after the dotted // keys inside it ({a.b = 1}) built it return m, nil } // show is t as an error names it; a number's text, which the lexer leaves out, // read back from the input func (p *Parser) show(t Token) string { if t.Type == TokenInteger || t.Type == TokenFloat { t.Literal = string(p.lexer.input[t.Pos:t.End]) } return t.String() } // adjacent reports b starting where a ends func adjacent(a, b Token) bool { return b.Pos == a.End } // brief quotes s for an error, cut to 32 bytes: a literal can be megabytes func brief(s string) string { if len(s) > 32 { return fmt.Sprintf("%q... (%d bytes)", s[:32], len(s)) } return fmt.Sprintf("%q", s) } // numCause drops strconv's copy of the literal, which brief already shows func numCause(err error) error { if ne, ok := errors.AsType[*strconv.NumError](err); ok { return ne.Err } return err } // Keep the package's decimal-key restriction independent of numeric range. func numericKey(s string) bool { if len(s) > 0 && (s[0] == '+' || s[0] == '-') { s = s[1:] } if s == "" { return false } for _, c := range s { if !isDigit(c) { return false } } return true }