v0.2.0 edit a document in place, comments kept; name the keys a struct does not take

.github/workflows/test.yml

@@ -16,3 +16,4 @@ jobs:
       - run: go vet ./...
       - run: go test -run '^$' -fuzz '^FuzzRoundTrip$' -fuzztime=15s -parallel=2
       - run: go test -run '^$' -fuzz '^FuzzIntegerConversions$' -fuzztime=10s -parallel=2
+      - run: go test -run '^$' -fuzz '^FuzzDocument$' -fuzztime=10s -parallel=2

README.md

@@ -74,12 +74,42 @@ change it. Errors quote keys and cut long literals, and never repeat a string
 value. `Marshal` writes a table inline once its header path would pass 256
 bytes, so output stays linear in depth.
 
-Comments and original formatting are not preserved by `Parse`/`Marshal`. A
+Comments and original formatting are not preserved by `Parse`/`Marshal`; a
-configuration editor can scan the original bytes with `NewLexer` and retain
+`Document` edits a file in place instead.
-`TokenComment.Literal` values, then prefix each with `#` and write them as a
+
-preamble before the marshaled data. This preserves comment text (including
+## Editing a document in place
-inline comments), but not its original location. Hashes inside strings are not
+
-comment tokens. Stop scanning on `TokenEOF` or `TokenError`.
+```go
+doc, err := toml.ParseDocument(data)
+err = doc.Decode(&settings)
+unknown := doc.Unmapped(&settings)         // []toml.Key no field takes
+removed, err := doc.Delete(toml.Key{"play", "effects"})
+err = doc.Set(toml.Key{"play", "sfx"}, false)
+data = doc.Bytes()
+```
+
+`ParseDocument` parses as `Parse` does and keeps the bytes. An edit rewrites only
+the statements it names, so comments, blank lines, order and the spelling of
+every other value stay as written; the result is parsed again, and an edit it
+would not parse after is refused with the document unchanged.
+
+- `Set` replaces the value of a key the document sets, keeping the rest of its
+  line and its comment. A key it lacks goes after the last key of its table,
+  spelled as that key is (`play.mouse` beside `play.hud`), else after the table's
+  header, else in a `[table]` appended at the end. A key under an array of
+  tables, inside an inline table or array, or naming a table fails. The value is
+  encoded as `Marshal` writes one.
+- `Delete` removes the key and every key under it: each statement setting one and
+  each header naming one, in every table of an array of tables. It reports
+  whether it removed anything; a key inside an inline value fails.
+- `Unmapped` names the keys no field of the destination takes, matched as `Decode`
+  matches them (tags, then field names; unexported and `-` fields take nothing),
+  each once and sorted. A map or interface destination takes every key under it;
+  a key under an array of tables is named without its index.
+- `Key` is a dotted path, one segment per part; `Key.String` spells it as a
+  document would, quoting segments that need it.
+
+A token's `Pos` and `End` are its byte offsets in the input.
 
 ## Deliberate subset and deviations
 

document.go

@@ -0,0 +1,305 @@
+package toml
+
+import (
+	"bytes"
+	"cmp"
+	"fmt"
+	"reflect"
+	"slices"
+	"strings"
+)
+
+// Key is a dotted key path, one segment per part: [a."b.c"] is Key{"a", "b.c"}
+type Key []string
+
+// String spells the key as a document writes it, quoting segments that need it
+func (k Key) String() string {
+	parts := make([]string, len(k))
+	for i, s := range k {
+		if t, err := keyText(s); err == nil {
+			parts[i] = t
+		} else {
+			var e encoder
+			e.string(s)
+			parts[i] = e.buf.String()
+		}
+	}
+	return strings.Join(parts, ".")
+}
+
+// under reports k is p or a key inside it
+func (k Key) under(p Key) bool { return len(k) >= len(p) && slices.Equal(k[:len(p)], p) }
+
+// Document is a parsed document that keeps its bytes. An edit rewrites only the
+// statements it names, so comments, blank lines, order and the spelling of every
+// other value stay as written, and the result is parsed again before it stands.
+type Document struct {
+	data  []byte
+	tree  map[string]any
+	stmts []statement
+}
+
+// statement is a table header or a key and its value, as the lines it spans: from
+// the start of its line through its newline, a comment after it on that line
+// included. A key is its path relative to scope, the header it sits under.
+type statement struct {
+	start, end int
+	scope, key Key
+	header     bool
+	array      bool   // an [[array]] header
+	value      [2]int // a key's value bytes
+}
+
+func (s statement) path() Key {
+	if s.header {
+		return s.key
+	}
+	return append(slices.Clip(s.scope), s.key...)
+}
+
+// ParseDocument parses data as Parse does and keeps it for editing; an edit
+// writes a new slice, never into data
+func ParseDocument(data []byte) (*Document, error) {
+	tree, err := NewParser(data).Parse()
+	if err != nil {
+		return nil, err
+	}
+	return &Document{data: data, tree: tree, stmts: statements(data)}, nil
+}
+
+// statements splits a document the parser accepted into its statements
+func statements(data []byte) []statement {
+	lx := NewLexer(data)
+	t := lx.NextToken()
+	key := func() Key {
+		var k Key
+		for {
+			k = append(k, t.Literal)
+			if t = lx.NextToken(); t.Type != TokenDot {
+				return k
+			}
+			t = lx.NextToken()
+		}
+	}
+	// lineEnd takes a trailing comment and the newline, leaving t past them
+	lineEnd := func() int {
+		if t.Type == TokenComment {
+			t = lx.NextToken()
+		}
+		end := t.End
+		if t.Type != TokenEOF {
+			t = lx.NextToken()
+		}
+		return end
+	}
+	var (
+		out   []statement
+		scope Key
+	)
+	for t.Type != TokenEOF {
+		s := statement{start: bytes.LastIndexByte(data[:t.Pos], '\n') + 1, scope: scope}
+		switch t.Type {
+		case TokenLBracket:
+			s.header, t = true, lx.NextToken()
+			if t.Type == TokenLBracket {
+				s.array, t = true, lx.NextToken()
+			}
+			s.key = key()
+			for t.Type == TokenRBracket {
+				t = lx.NextToken()
+			}
+			s.scope, scope = nil, s.key
+		case TokenIdent, TokenString:
+			s.key = key()
+			t = lx.NextToken() // past =
+			s.value[0] = t.Pos
+			for depth := 0; ; {
+				switch t.Type {
+				case TokenLBracket, TokenLBrace:
+					depth++
+				case TokenRBracket, TokenRBrace:
+					depth--
+				}
+				if t.Type != TokenNewline && t.Type != TokenComment {
+					s.value[1] = t.End
+				}
+				if t = lx.NextToken(); depth == 0 && (t.Type == TokenNewline || t.Type == TokenComment || t.Type == TokenEOF) {
+					break
+				}
+			}
+		default: // a blank or comment line
+			t = lx.NextToken()
+			continue
+		}
+		s.end = lineEnd()
+		out = append(out, s)
+	}
+	return out
+}
+
+// Bytes is the document as it now reads; the caller must not change it
+func (d *Document) Bytes() []byte { return d.data }
+
+// Tree is the document's values as Parse returns them, which Decode reads
+func (d *Document) Tree() map[string]any { return d.tree }
+
+// Decode stores the document's values in v as Decode does
+func (d *Document) Decode(v any) error { return Decode(d.tree, v) }
+
+// Unmapped names the keys of the document that no field of v's type takes, as
+// Decode, which ignores them, would match them; each once and sorted. A key under
+// a map or interface destination is taken, and one under an array of tables is
+// named without its index.
+func (d *Document) Unmapped(v any) []Key {
+	seen := make(map[string]Key)
+	unmapped(d.tree, reflect.TypeOf(v), nil, seen, 0)
+	out := make([]Key, 0, len(seen))
+	for _, k := range seen {
+		out = append(out, k)
+	}
+	slices.SortFunc(out, func(a, b Key) int { return slices.Compare(a, b) })
+	return out
+}
+
+func unmapped(data any, t reflect.Type, at Key, seen map[string]Key, depth int) {
+	for t != nil && t.Kind() == reflect.Pointer {
+		t = t.Elem()
+	}
+	if t == nil || depth > maxValueDepth {
+		return
+	}
+	switch t.Kind() {
+	case reflect.Struct:
+		m, ok := data.(map[string]any)
+		if !ok {
+			return
+		}
+		fields := make(map[string]reflect.Type, t.NumField())
+		for i := range t.NumField() {
+			f := t.Field(i)
+			name, _, _ := strings.Cut(f.Tag.Get("toml"), ",")
+			if f.IsExported() && name != "-" {
+				fields[cmp.Or(name, f.Name)] = f.Type
+			}
+		}
+		for k, v := range m {
+			key := append(slices.Clip(at), k)
+			if ft, ok := fields[k]; ok {
+				unmapped(v, ft, key, seen, depth+1)
+			} else {
+				seen[key.String()] = key
+			}
+		}
+	case reflect.Map:
+		if m, ok := data.(map[string]any); ok {
+			for k, v := range m {
+				unmapped(v, t.Elem(), append(slices.Clip(at), k), seen, depth+1)
+			}
+		}
+	case reflect.Slice, reflect.Array:
+		switch s := data.(type) {
+		case []any:
+			for _, v := range s {
+				unmapped(v, t.Elem(), at, seen, depth+1)
+			}
+		case []map[string]any:
+			for _, v := range s {
+				unmapped(v, t.Elem(), at, seen, depth+1)
+			}
+		}
+	}
+}
+
+// Delete removes key and every key under it: each statement setting one and each
+// table header naming one, in every table of an array of tables, every other line
+// kept. It reports whether anything was removed; a key inside an inline value, a
+// table or array written as a value, cannot be edited and fails.
+func (d *Document) Delete(key Key) (bool, error) {
+	var out []byte
+	at := 0
+	for _, s := range d.stmts {
+		p := s.path()
+		if !s.header && len(p) < len(key) && key.under(p) && d.inline(s) {
+			return false, fmt.Errorf("key %s is inside the value of %s", key, p)
+		}
+		if p.under(key) {
+			out, at = append(out, d.data[at:s.start]...), s.end
+		}
+	}
+	if at == 0 {
+		return false, nil
+	}
+	return true, d.replace(append(out, d.data[at:]...))
+}
+
+// inline reports a key's value is an inline table or an array
+func (d *Document) inline(s statement) bool {
+	c := d.data[s.value[0]]
+	return c == '{' || c == '['
+}
+
+// Set gives key the value v, encoded as Marshal writes a value. A key the document
+// sets has its value replaced, the rest of its line kept; a key it lacks goes after
+// the last key of its table, else after the table's header, else in a table
+// appended to the document. A key under an array of tables or inside an inline
+// value fails, and so does a value the document then would not parse with.
+func (d *Document) Set(key Key, v any) error {
+	if len(key) == 0 || v == nil {
+		return fmt.Errorf("a key and a value are needed")
+	}
+	var e encoder
+	if err := e.value(reflect.ValueOf(v), 0); err != nil {
+		return fmt.Errorf("key %s: %w", key, err)
+	}
+	value := e.buf.Bytes()
+	parent := key[:len(key)-1]
+	after, scope := -1, Key(nil) // where a missing key goes, and the scope it is spelled in
+	for _, s := range d.stmts {
+		p := s.path()
+		switch {
+		case s.header && s.array && key.under(p):
+			return fmt.Errorf("key %s is under the array of tables %s", key, p)
+		case !s.header && slices.Equal(p, key):
+			return d.replace(slices.Concat(d.data[:s.value[0]], value, d.data[s.value[1]:]))
+		case !s.header && len(p) < len(key) && key.under(p):
+			return fmt.Errorf("key %s is inside the value of %s", key, p)
+		case p.under(key):
+			return fmt.Errorf("key %s is a table", key)
+		case s.header && slices.Equal(p, parent):
+			after, scope = s.end, p
+		case !s.header && len(p) > len(parent) && p.under(parent) && parent.under(s.scope):
+			if !slices.Equal(scope, parent) || slices.Equal(s.scope, parent) {
+				after, scope = s.end, s.scope
+			}
+		}
+	}
+	head := ""
+	switch {
+	case after >= 0:
+	case len(parent) == 0: // a root key goes before the first header
+		after = len(d.data)
+		if i := slices.IndexFunc(d.stmts, func(s statement) bool { return s.header }); i >= 0 {
+			after = d.stmts[i].start
+		}
+	default:
+		after, scope, head = len(d.data), parent, "["+parent.String()+"]\n"
+		if after > 0 {
+			head = "\n" + head
+		}
+	}
+	line := slices.Concat([]byte(head+key[len(scope):].String()+" = "), value, []byte("\n"))
+	if after > 0 && d.data[after-1] != '\n' {
+		line = slices.Concat([]byte("\n"), line)
+	}
+	return d.replace(slices.Concat(d.data[:after], line, d.data[after:]))
+}
+
+// replace makes data the document once it parses
+func (d *Document) replace(data []byte) error {
+	next, err := ParseDocument(data)
+	if err != nil {
+		return fmt.Errorf("edit refused: %w", err)
+	}
+	*d = *next
+	return nil
+}

document_test.go

@@ -0,0 +1,268 @@
+package toml
+
+import (
+	"fmt"
+	"slices"
+	"testing"
+)
+
+const settingsDoc = `# mine
+play.hud = true # root dotted
+
+[paths]
+log = "l" # beside the file
+
+[play]
+# kept
+mouse = false # keys
+volume = 0.3
+
+# about later
+[later] # a newer build's
+key = 1
+[later.sub]
+deep = [
+  1, # one
+  2,
+]
+
+[audio]
+buffer_ms = 20
+`
+
+// TestDocumentEditsOnlyWhatItNames: a set value keeps the rest of its line, a
+// missing key goes after its table's last key in that key's spelling, a missing
+// table is appended, a delete takes a key's lines or a table's header with every
+// key under it, and every other byte, comments and blank lines, stays.
+func TestDocumentEditsOnlyWhatItNames(t *testing.T) {
+	d, err := ParseDocument([]byte(settingsDoc))
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, edit := range []func() error{
+		func() error { return d.Set(Key{"paths", "log"}, "logs") },
+		func() error { return d.Set(Key{"play", "sfx"}, false) },
+		func() error { return d.Set(Key{"presentation", "cell"}, 14.0) },
+		func() error { return d.Set(Key{"audio", "buffer ms"}, 30) },
+		func() error { return del(d, Key{"play", "volume"}) },
+		func() error { return del(d, Key{"later"}) },
+	} {
+		if err := edit(); err != nil {
+			t.Fatal(err)
+		}
+	}
+	want := `# mine
+play.hud = true # root dotted
+
+[paths]
+log = "logs" # beside the file
+
+[play]
+# kept
+mouse = false # keys
+sfx = false
+
+# about later
+
+[audio]
+buffer_ms = 20
+"buffer ms" = 30
+
+[presentation]
+cell = 14.0
+`
+	if got := string(d.Bytes()); got != want {
+		t.Fatalf("edited:\n%s\nwant:\n%s", got, want)
+	}
+	var s struct {
+		Play struct {
+			HUD   bool `toml:"hud"`
+			Mouse bool `toml:"mouse"`
+			Sfx   bool `toml:"sfx"`
+		} `toml:"play"`
+	}
+	s.Play.Sfx = true
+	if err := d.Decode(&s); err != nil || !s.Play.HUD || s.Play.Mouse || s.Play.Sfx {
+		t.Errorf("decoded %+v, %v", s, err)
+	}
+}
+
+func del(d *Document, k Key) error {
+	ok, err := d.Delete(k)
+	if err == nil && !ok {
+		err = fmt.Errorf("%s: nothing deleted", k)
+	}
+	return err
+}
+
+// TestSetSpellsAKeyAsItsTableDoes: with no header, a key joins the dotted keys that
+// made its table, spelled as they are.
+func TestSetSpellsAKeyAsItsTableDoes(t *testing.T) {
+	d, err := ParseDocument([]byte("play.hud = true\n\n[audio]\nbuffer_ms = 20\n"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := d.Set(Key{"play", "mouse"}, false); err != nil {
+		t.Fatal(err)
+	}
+	if got, want := string(d.Bytes()), "play.hud = true\nplay.mouse = false\n\n[audio]\nbuffer_ms = 20\n"; got != want {
+		t.Fatalf("got:\n%s\nwant:\n%s", got, want)
+	}
+}
+
+// TestDocumentRefusesAnEditItCannotPlace: under an array of tables, inside an
+// inline value, or over a table, an edit fails and the document stays as it was.
+func TestDocumentRefusesAnEditItCannotPlace(t *testing.T) {
+	doc := "x = 1\nm = { a = 1 }\n[t]\nk = 1\n[[arr]]\nn = 1\n"
+	d, err := ParseDocument([]byte(doc))
+	if err != nil {
+		t.Fatal(err)
+	}
+	for name, edit := range map[string]func() error{
+		"array of tables": func() error { return d.Set(Key{"arr", "n"}, 2) },
+		"inline set":      func() error { return d.Set(Key{"m", "b"}, 2) },
+		"inline delete":   func() error { _, err := d.Delete(Key{"m", "a"}); return err },
+		"table":           func() error { return d.Set(Key{"t"}, 2) },
+		"under a value":   func() error { return d.Set(Key{"x", "y"}, 2) },
+	} {
+		if err := edit(); err == nil {
+			t.Errorf("%s: the edit was taken", name)
+		}
+	}
+	if string(d.Bytes()) != doc {
+		t.Fatalf("a refused edit changed the document:\n%s", d.Bytes())
+	}
+}
+
+// TestUnmappedNamesWhatDecodeIgnores: keys no field takes, by tag or field name,
+// unexported and "-" fields included, are named once each, under arrays of tables
+// without their index; a map or interface destination takes every key under it.
+func TestUnmappedNamesWhatDecodeIgnores(t *testing.T) {
+	doc := `top = 1
+skip = 2
+hidden = 3
+Named = 4
+[sub]
+ok = 1
+bad = 2
+[sub.deeper]
+x = 1
+[free]
+anything = 1
+[anyv]
+v = { w = 1 }
+[[items]]
+id = 1
+odd = 1
+[[items]]
+id = 2
+odd = 2
+[byname.one]
+ok = 1
+nope = 1
+`
+	type item struct {
+		ID int `toml:"id"`
+	}
+	type sub struct {
+		OK int `toml:"ok,omitempty"`
+	}
+	var v struct {
+		Top    int `toml:"top"`
+		Skip   int `toml:"-"`
+		hidden int
+		Named  int            `toml:",omitempty"`
+		Sub    *sub           `toml:"sub"`
+		Free   map[string]any `toml:"free"`
+		Anyv   any            `toml:"anyv"`
+		Items  []item         `toml:"items"`
+		ByName map[string]sub `toml:"byname"`
+	}
+	d, err := ParseDocument([]byte(doc))
+	if err != nil {
+		t.Fatal(err)
+	}
+	var got []string
+	for _, k := range d.Unmapped(&v) {
+		got = append(got, k.String())
+	}
+	want := []string{"byname.one.nope", "hidden", "items.odd", "skip", "sub.bad", "sub.deeper"}
+	if !slices.Equal(got, want) {
+		t.Fatalf("unmapped %q, want %q", got, want)
+	}
+}
+
+// TestKeyStringQuotesWhatABareKeyCannotHold
+func TestKeyStringQuotesWhatABareKeyCannotHold(t *testing.T) {
+	if got := (Key{"a", "b.c", "d e", "7"}).String(); got != `a."b.c"."d e"."7"` {
+		t.Fatalf("got %s", got)
+	}
+}
+
+// FuzzDocument: a document that parses keeps its bytes; deleting any key it sets
+// leaves a shorter document that parses without that key, and setting one or a
+// key beside it, where the edit is taken, leaves it holding the value set.
+func FuzzDocument(f *testing.F) {
+	for _, s := range []string{
+		settingsDoc,
+		"a = 1\n[t]\nk = [1,\n2] # c\n[[r]]\nx = 1\n[[r]]\nx = 2\n",
+		"t.a.b = { c = 1 }\n# end",
+		"\"q.k\" = 1\r\n[\"x y\".z]\r\nv = \"#\"\r\n",
+	} {
+		f.Add([]byte(s))
+	}
+	f.Fuzz(func(t *testing.T, data []byte) {
+		d, err := ParseDocument(data)
+		if err != nil {
+			return
+		}
+		if string(d.Bytes()) != string(data) {
+			t.Fatal("a parsed document changed its bytes")
+		}
+		for _, s := range d.stmts {
+			c, _ := ParseDocument(data)
+			ok, err := c.Delete(s.path())
+			if err != nil {
+				continue
+			}
+			if !ok {
+				t.Fatalf("delete of %s, which the document sets, removed nothing", s.path())
+			}
+			if has(c.Tree(), s.path()) {
+				t.Fatalf("%s still set after its delete:\n%s", s.path(), c.Bytes())
+			}
+			if len(c.Bytes()) >= len(data) {
+				t.Fatalf("delete of %s did not shorten the document", s.path())
+			}
+		}
+		for _, s := range d.stmts {
+			for _, k := range []Key{s.path(), append(s.path()[:len(s.path())-1:len(s.path())-1], "added")} {
+				c, _ := ParseDocument(data)
+				if err := c.Set(k, int64(7)); err == nil && value(c.Tree(), k) != int64(7) {
+					t.Fatalf("set %s took but reads %v:\n%s", k, value(c.Tree(), k), c.Bytes())
+				}
+			}
+		}
+	})
+}
+
+func has(tree map[string]any, k Key) bool { return value(tree, k) != nil }
+
+// value is what k holds in tree, in the last table of an array of tables
+func value(tree map[string]any, k Key) any {
+	var cur any = tree
+	for _, s := range k {
+		m, ok := cur.(map[string]any)
+		if !ok {
+			if a, ok := cur.([]map[string]any); ok && len(a) > 0 {
+				m = a[len(a)-1]
+			} else {
+				return nil
+			}
+		}
+		if cur, ok = m[s]; !ok {
+			return nil
+		}
+	}
+	return cur
+}

lexer.go

@@ -23,7 +23,13 @@ func NewLexer(input []byte) *Lexer {
 
 func (l *Lexer) NextToken() Token {
 	l.skipWhitespace()
+	start := l.pos
+	t := l.next()
+	t.Pos, t.End = start, l.pos
+	return t
+}
 
+func (l *Lexer) next() Token {
 	if l.pos >= len(l.input) {
 		return l.newToken(TokenEOF, "")
 	}

token.go

@@ -30,12 +30,13 @@ const (
 	TokenNewline  // \n
 )
 
-// Token represents a lexical token
+// Token represents a lexical token; Pos and End are its byte offsets in the input
 type Token struct {
-	Type    TokenType
+	Type     TokenType
-	Literal string
+	Literal  string
-	Line    int
+	Line     int
-	Col     int
+	Col      int
+	Pos, End int
 }
 
 func (t Token) String() string {