v0.1.1 decode hardening, round trip support, doc and test update
Test / test (push) Canceled after 0s

This commit is contained in:
lixen committed 2026-09-28 14:05:12 -04:00
1 parent c8e23ff859
commit b59d068ff6
15 files changed
+874 -651

No files matched your search

+225 -381
View File
@@ -2,460 +2,304 @@ package toml
import (
"bytes"
"cmp"
"fmt"
"math"
"reflect"
"sort"
"slices"
"strconv"
"strings"
"unicode/utf8"
)
// Marshal returns the TOML encoding of v
//
// Marshal supports struct and map types as the root object
// It follows standard TOML formatting:
// - Comments and whitespace are not preserved from original decoding
// - Dates are not supported (as per requirements).
// - Nil pointers are skipped
// - Unexported fields are skipped
// - Fields with `omitempty` are skipped if zero
// - Struct fields/Map keys are sorted alphabetically for determinism
// - Fully numerical keys are rejected
// Marshal encodes a struct or string-keyed map with deterministic key order.
// Nil pointer/interface fields are omitted; nil pointer/interface array elements,
// cycles/excessive nesting,
// non-finite floats, and integers outside TOML's signed 64-bit range error.
// Comments and whitespace are not retained. Datetimes are not supported.
func Marshal(v any) ([]byte, error) {
val := reflect.ValueOf(v)
// Dereference pointer if necessary
if val.Kind() == reflect.Ptr {
if val.IsNil() {
return nil, fmt.Errorf("marshal: cannot marshal nil pointer")
}
val = val.Elem()
}
// Root must be a Table (Struct or Map)
if val.Kind() != reflect.Struct && val.Kind() != reflect.Map {
return nil, fmt.Errorf("marshal: root must be struct or map, got %v", val.Kind())
}
buf := new(bytes.Buffer)
enc := &encoder{w: buf}
if err := enc.encodeTable(val, ""); err != nil {
var e encoder
if err := e.table(reflect.ValueOf(v), "", 0); err != nil {
return nil, err
}
return buf.Bytes(), nil
return e.buf.Bytes(), nil
}
type encoder struct {
w *bytes.Buffer
type encoder struct{ buf bytes.Buffer }
type entry struct {
key string
value reflect.Value
}
// encodeTable writes the fields of a struct or map.
// It uses a two-pass approach:
// 1. Write all scalar values (primitives, inline arrays).
// 2. Recurse and write nested tables (structs, maps, arrays of tables).
// This ensures valid TOML where keys are defined before sub-tables.
func (e *encoder) encodeTable(rv reflect.Value, prefix string) error {
// 1. Gather keys
keys, err := e.getSortedKeys(rv)
func indirect(v reflect.Value) (reflect.Value, error) {
for depth := 0; v.IsValid() && (v.Kind() == reflect.Interface || v.Kind() == reflect.Pointer); depth++ {
if depth > maxValueDepth {
return reflect.Value{}, fmt.Errorf("pointer nesting exceeds %d", maxValueDepth)
}
if v.IsNil() {
return reflect.Value{}, nil
}
v = v.Elem()
}
return v, nil
}
func entries(v reflect.Value) ([]entry, error) {
v, err := indirect(v)
if err != nil {
return nil, err
}
if !v.IsValid() {
return nil, fmt.Errorf("nil table")
}
var out []entry
switch v.Kind() {
case reflect.Map:
if v.Type().Key().Kind() != reflect.String {
return nil, fmt.Errorf("map keys must be strings")
}
out = make([]entry, 0, v.Len())
it := v.MapRange()
for it.Next() {
out = append(out, entry{it.Key().String(), it.Value()})
}
case reflect.Struct:
t := v.Type()
out = make([]entry, 0, t.NumField())
for i := 0; i < t.NumField(); i++ {
f := t.Field(i)
if !f.IsExported() {
continue
}
parts := strings.Split(f.Tag.Get("toml"), ",")
if parts[0] == "-" {
continue
}
key := f.Name
if parts[0] != "" {
key = parts[0]
}
if slices.Contains(parts[1:], "omitempty") && isEmptyValue(v.Field(i)) {
continue
}
out = append(out, entry{key, v.Field(i)})
}
default:
return nil, fmt.Errorf("expected struct or map, got %s", v.Kind())
}
slices.SortFunc(out, func(a, b entry) int { return cmp.Compare(a.key, b.key) })
result := out[:0]
previous := ""
for i, x := range out {
if _, err := keyText(x.key); err != nil {
return nil, err
}
if i > 0 && previous == x.key {
return nil, fmt.Errorf("duplicate TOML key %q", x.key)
}
previous = x.key
value, err := indirect(x.value)
if err != nil {
return nil, err
}
if value.IsValid() {
result = append(result, entry{x.key, value})
}
}
return result, nil
}
func (e *encoder) table(v reflect.Value, prefix string, depth int) error {
if depth > maxValueDepth {
return fmt.Errorf("encode nesting exceeds %d (possible cycle)", maxValueDepth)
}
fields, err := entries(v)
if err != nil {
return err
}
// 2. Separation: Identify which keys are scalars (printed now) vs tables (printed later)
var scalars []string
var tables []string
for _, k := range keys {
fieldVal := e.resolveValue(rv, k)
if !fieldVal.IsValid() {
continue // Skip invalid/nil
}
// Check if we should skip (omitempty, unexported handled in getSortedKeys)
if e.shouldSkip(rv, k, fieldVal) {
var tables []entry
for _, x := range fields {
if isTable(x.value) || isTableArray(x.value) {
tables = append(tables, x)
continue
}
// isTable validates slice homogeneity
isTab, err := e.isTable(fieldVal)
if err != nil {
return fmt.Errorf("key %q: %w", e.getKeyName(rv, k), err)
}
if isTab {
tables = append(tables, k)
} else {
scalars = append(scalars, k)
key, _ := keyText(x.key)
e.buf.WriteString(key + " = ")
if err := e.value(x.value, depth+1); err != nil {
return fmt.Errorf("key %q: %w", x.key, err)
}
e.buf.WriteByte('\n')
}
// 3. Pass 1: Write Scalars
for _, k := range scalars {
val := e.resolveValue(rv, k)
keyName := e.getKeyName(rv, k)
if err := e.writeKey(keyName); err != nil {
return err
}
if _, err := e.w.WriteString(" = "); err != nil {
return err
}
if err := e.encodeValue(val); err != nil {
return fmt.Errorf("key %q: %w", keyName, err)
}
e.w.WriteString("\n")
}
// 4. Pass 2: Write Tables
for _, k := range tables {
val := e.resolveValue(rv, k)
keyName := e.getKeyName(rv, k)
// Determine full path for header
fullKey := keyName
for _, x := range tables {
key, _ := keyText(x.key)
if prefix != "" {
fullKey = prefix + "." + keyName
key = prefix + "." + key
}
// Handle specific table types
switch val.Kind() {
case reflect.Struct, reflect.Map:
// [header]
e.w.WriteString("\n")
e.w.WriteString("[" + fullKey + "]\n")
if err := e.encodeTable(val, fullKey); err != nil {
if isTable(x.value) {
e.buf.WriteString("\n[" + key + "]\n")
if err := e.table(x.value, key, depth+1); err != nil {
return err
}
case reflect.Slice, reflect.Array:
// [[header]]
for i := 0; i < val.Len(); i++ {
elem := val.Index(i)
// Dereference pointer elements in slice
if elem.Kind() == reflect.Ptr {
if elem.IsNil() {
continue
}
elem = elem.Elem()
}
e.w.WriteString("\n")
e.w.WriteString("[[" + fullKey + "]]\n")
if err := e.encodeTable(elem, fullKey); err != nil {
} else {
for i := 0; i < x.value.Len(); i++ {
e.buf.WriteString("\n[[" + key + "]]\n")
if err := e.table(x.value.Index(i), key, depth+1); err != nil {
return err
}
}
}
}
return nil
}
// encodeValue writes a single primitive value or inline array
func (e *encoder) encodeValue(v reflect.Value) error {
func isTable(v reflect.Value) bool { return v.Kind() == reflect.Map || v.Kind() == reflect.Struct }
func isTableArray(v reflect.Value) bool {
if v.Kind() != reflect.Slice && v.Kind() != reflect.Array || v.Len() == 0 {
return false
}
for i := 0; i < v.Len(); i++ {
x, err := indirect(v.Index(i))
if err != nil || !x.IsValid() || !isTable(x) {
return false
}
}
return true
}
func (e *encoder) value(v reflect.Value, depth int) error {
if depth > maxValueDepth {
return fmt.Errorf("encode nesting exceeds %d (possible cycle)", maxValueDepth)
}
v, err := indirect(v)
if err != nil {
return err
}
if !v.IsValid() {
return fmt.Errorf("nil array element has no TOML representation")
}
switch v.Kind() {
case reflect.Bool:
if v.Bool() {
e.w.WriteString("true")
} else {
e.w.WriteString("false")
}
e.buf.WriteString(strconv.FormatBool(v.Bool()))
case reflect.String:
e.encodeString(v.String())
if !utf8.ValidString(v.String()) {
return fmt.Errorf("invalid UTF-8 string")
}
e.string(v.String())
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
e.w.WriteString(strconv.FormatInt(v.Int(), 10))
e.buf.WriteString(strconv.FormatInt(v.Int(), 10))
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
e.w.WriteString(strconv.FormatUint(v.Uint(), 10))
if v.Uint() > math.MaxInt64 {
return fmt.Errorf("unsigned integer %d exceeds TOML int64 range", v.Uint())
}
e.buf.WriteString(strconv.FormatUint(v.Uint(), 10))
case reflect.Float32, reflect.Float64:
f := v.Float()
// NaN/Inf have no TOML representation; emitting them produces
// output the parser rejects
if math.IsNaN(f) || math.IsInf(f, 0) {
return fmt.Errorf("cannot encode %v as TOML float", f)
return fmt.Errorf("non-finite float")
}
// bitSize matches kind (float32 emitted noise digits);
// 'g' avoids ~100-digit 'f' expansions for large exponents
bits := 64
if v.Kind() == reflect.Float32 {
bits = 32
s := strconv.FormatFloat(f, 'g', -1, v.Type().Bits())
if !strings.ContainsAny(s, ".eE") {
s += ".0"
}
str := strconv.FormatFloat(f, 'g', -1, bits)
if !strings.ContainsAny(str, ".eE") {
str += ".0"
}
e.w.WriteString(str)
e.buf.WriteString(s)
case reflect.Slice, reflect.Array:
// Inline array: [1, 2, "3"]
e.w.WriteString("[")
e.buf.WriteByte('[')
for i := 0; i < v.Len(); i++ {
if i > 0 {
e.w.WriteString(", ")
e.buf.WriteString(", ")
}
if err := e.encodeValue(v.Index(i)); err != nil {
if err := e.value(v.Index(i), depth+1); err != nil {
return err
}
}
e.w.WriteString("]")
case reflect.Interface:
if v.IsNil() {
return nil // Should be handled by caller usually
e.buf.WriteByte(']')
case reflect.Struct, reflect.Map:
fields, err := entries(v)
if err != nil {
return err
}
return e.encodeValue(v.Elem())
e.buf.WriteByte('{')
for i, x := range fields {
if i > 0 {
e.buf.WriteString(", ")
}
key, _ := keyText(x.key)
e.buf.WriteString(key + " = ")
if err := e.value(x.value, depth+1); err != nil {
return err
}
}
e.buf.WriteByte('}')
default:
return fmt.Errorf("unsupported type: %v", v.Kind())
return fmt.Errorf("unsupported type %s", v.Type())
}
return nil
}
// --- Helpers ---
// getSortedKeys returns all field names (struct) or keys (map) sorted
func (e *encoder) getSortedKeys(rv reflect.Value) ([]string, error) {
var keys []string
if rv.Kind() == reflect.Map {
for _, key := range rv.MapKeys() {
if key.Kind() != reflect.String {
return nil, fmt.Errorf("map key must be string, got %v", key.Kind())
}
keys = append(keys, key.String())
}
} else if rv.Kind() == reflect.Struct {
typ := rv.Type()
for i := 0; i < rv.NumField(); i++ {
field := typ.Field(i)
// Skip unexported
if field.PkgPath != "" {
continue
}
// Skip if tag is "-"
tag := field.Tag.Get("toml")
if tag == "-" {
continue
}
keys = append(keys, field.Name)
}
func keyText(s string) (string, error) {
if !utf8.ValidString(s) {
return "", fmt.Errorf("invalid UTF-8 key")
}
// Sort by emitted key (tag-resolved), not Go field name.
// Identity for maps (getKeyName returns the key itself).
sort.Slice(keys, func(i, j int) bool {
return e.getKeyName(rv, keys[i]) < e.getKeyName(rv, keys[j])
})
return keys, nil
if numericKey(s) {
return "", fmt.Errorf("numeric keys are forbidden: %q", s)
}
if isBareKey(s) {
return s, nil
}
var e encoder
e.string(s)
return e.buf.String(), nil
}
// resolveValue extracts the value from struct field or map key
// It also handles interface unwrapping for map[string]any.
func (e *encoder) resolveValue(container reflect.Value, key string) reflect.Value {
var val reflect.Value
if container.Kind() == reflect.Map {
val = container.MapIndex(reflect.ValueOf(key))
} else {
val = container.FieldByName(key)
}
// Unwrap interface if needed
if val.Kind() == reflect.Interface && !val.IsNil() {
val = val.Elem()
}
// Dereference pointer if needed (but keep nil ptrs for checking)
if val.Kind() == reflect.Ptr && !val.IsNil() {
val = val.Elem()
}
return val
}
// getKeyName resolves the TOML key name (handles struct tags)
// For maps, the key name is the key itself
func (e *encoder) getKeyName(container reflect.Value, realName string) string {
if container.Kind() == reflect.Map {
return realName
}
// Struct: lookup tag
field, _ := container.Type().FieldByName(realName)
tag := field.Tag.Get("toml")
if tag != "" {
parts := strings.Split(tag, ",")
if parts[0] != "" {
return parts[0]
func (e *encoder) string(s string) {
e.buf.WriteByte('"')
for _, r := range s {
switch r {
case '"':
e.buf.WriteString(`\"`)
case '\\':
e.buf.WriteString(`\\`)
case '\n':
e.buf.WriteString(`\n`)
case '\r':
e.buf.WriteString(`\r`)
case '\t':
e.buf.WriteString(`\t`)
case '\b':
e.buf.WriteString(`\b`)
case '\f':
e.buf.WriteString(`\f`)
default:
if r < 0x20 || r == 0x7f {
fmt.Fprintf(&e.buf, `\u%04X`, r)
} else {
e.buf.WriteRune(r)
}
}
}
return realName
e.buf.WriteByte('"')
}
// shouldSkip returns true if the field should be omitted (nil ptr, omitempty)
func (e *encoder) shouldSkip(container reflect.Value, realName string, val reflect.Value) bool {
// Skip nil pointers / interfaces
if (val.Kind() == reflect.Ptr || val.Kind() == reflect.Interface) && val.IsNil() {
return true
}
// Maps don't have tags, so we only skip nil values
if container.Kind() == reflect.Map {
func isBareKey(s string) bool {
if s == "" || s == "true" || s == "false" {
return false
}
// Structs: check omitempty
field, _ := container.Type().FieldByName(realName)
tag := field.Tag.Get("toml")
if strings.Contains(tag, "omitempty") && isEmptyValue(val) {
return true
if s[0] >= '0' && s[0] <= '9' || len(s) > 1 && s[0] == '-' && s[1] >= '0' && s[1] <= '9' {
return false
}
return false
}
// isTable determines if a value renders as [Table] / [[Array of Tables]].
// Slices must be homogeneous: mixing table and scalar elements is an error
// (parser accepts such arrays; encoder cannot represent them — documented
// limitation). Nil pointer elements are allowed in table slices and skipped
// at emission; nil interface elements are an error.
func (e *encoder) isTable(v reflect.Value) (bool, error) {
if v.Kind() == reflect.Interface {
v = v.Elem()
}
if v.Kind() == reflect.Ptr {
v = v.Elem()
}
switch v.Kind() {
case reflect.Struct, reflect.Map:
return true, nil
case reflect.Slice, reflect.Array:
tables, scalars := 0, 0
for i := 0; i < v.Len(); i++ {
elem := v.Index(i)
if elem.Kind() == reflect.Interface {
if elem.IsNil() {
return false, fmt.Errorf("nil element at index %d", i)
}
elem = elem.Elem()
}
if elem.Kind() == reflect.Ptr {
if elem.IsNil() {
tables++ // nil pointers are only legal in [[table]] context (skipped)
continue
}
elem = elem.Elem()
}
switch elem.Kind() {
case reflect.Struct, reflect.Map:
tables++
default:
scalars++
}
for _, r := range s {
if !isAlpha(r) && !isDigit(r) && r != '_' && r != '-' {
return false
}
if tables > 0 && scalars > 0 {
return false, fmt.Errorf("mixed table/scalar elements in array")
}
return tables > 0, nil
}
return false, nil
return true
}
func isEmptyValue(v reflect.Value) bool {
switch v.Kind() {
case reflect.Array, reflect.Map, reflect.Slice, reflect.String:
return v.Len() == 0
case reflect.Bool:
return !v.Bool()
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return v.Int() == 0
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
return v.Uint() == 0
case reflect.Float32, reflect.Float64:
return v.Float() == 0
case reflect.Interface, reflect.Ptr:
return v.IsNil()
default:
return v.IsZero()
}
return false
}
func (e *encoder) writeKey(s string) error {
if isBareKey(s) {
_, err := e.w.WriteString(s)
return err
}
e.encodeString(s)
return nil
}
func (e *encoder) encodeString(s string) {
e.w.WriteString("\"")
for _, r := range s {
switch r {
case '"':
e.w.WriteString(`\"`)
case '\\':
e.w.WriteString(`\\`)
case '\n':
e.w.WriteString(`\n`)
case '\r':
e.w.WriteString(`\r`)
case '\t':
e.w.WriteString(`\t`)
case '\b':
e.w.WriteString(`\b`)
case '\f':
e.w.WriteString(`\f`)
default:
if r < 0x20 || r == 0x7F {
e.w.WriteString(fmt.Sprintf(`\u%04X`, r))
} else {
e.w.WriteRune(r)
}
}
}
e.w.WriteString("\"")
}
// isBareKey determines if a string can be written as a bare key
// It follows the TOML spec (A-Za-z0-9_-) but also respects the provided Lexer's behavior
// If the Lexer would interpret the string as a Number or Boolean, it must be quoted because the Parser expects TokenIdent (or TokenString) for keys
func isBareKey(s string) bool {
if s == "" {
return false
}
// 1. Valid bare key characters: A-Za-z0-9_-
for _, r := range s {
if !((r >= 'A' && r <= 'Z') || (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '_' || r == '-') {
return false
}
}
// 2. Avoid collision with Booleans
// Lexer emits TokenBool for these, Parser expects TokenIdent.
if s == "true" || s == "false" {
return false
}
// 3. Avoid collision with Numbers
// The provided Lexer triggers number parsing if the token starts with a digit,
// or a '-' followed by a digit.
// Since the Lexer never produces TokenIdent for these cases (it produces TokenInteger/Float),
// and the Parser rejects numeric tokens as keys, we must quote them.
c0 := s[0]
if c0 >= '0' && c0 <= '9' {
return false
}
if c0 == '-' && len(s) > 1 {
c1 := s[1]
if c1 >= '0' && c1 <= '9' {
return false
}
}
return true
}