From e2712e32a2196e124101f336cca6619032aad542 Mon Sep 17 00:00:00 2001 From: Lixen Wraith Date: Sat, 24 Oct 2026 20:45:44 -0400 Subject: [PATCH] v0.2.0 edit a document in place, comments kept; name the keys a struct does not take --- .github/workflows/test.yml | 1 + README.md | 42 ++++- document.go | 305 +++++++++++++++++++++++++++++++++++++ document_test.go | 268 ++++++++++++++++++++++++++++++++ lexer.go | 6 + token.go | 11 +- 6 files changed, 622 insertions(+), 11 deletions(-) create mode 100644 document.go create mode 100644 document_test.go diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ba4c09e..30db661 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,3 +16,4 @@ jobs: - run: go vet ./... - run: go test -run '^$' -fuzz '^FuzzRoundTrip$' -fuzztime=15s -parallel=2 - run: go test -run '^$' -fuzz '^FuzzIntegerConversions$' -fuzztime=10s -parallel=2 + - run: go test -run '^$' -fuzz '^FuzzDocument$' -fuzztime=10s -parallel=2 diff --git a/README.md b/README.md index bbbc49e..ba8dd94 100644 --- a/README.md +++ b/README.md @@ -74,12 +74,42 @@ change it. Errors quote keys and cut long literals, and never repeat a string value. `Marshal` writes a table inline once its header path would pass 256 bytes, so output stays linear in depth. -Comments and original formatting are not preserved by `Parse`/`Marshal`. A -configuration editor can scan the original bytes with `NewLexer` and retain -`TokenComment.Literal` values, then prefix each with `#` and write them as a -preamble before the marshaled data. This preserves comment text (including -inline comments), but not its original location. Hashes inside strings are not -comment tokens. Stop scanning on `TokenEOF` or `TokenError`. +Comments and original formatting are not preserved by `Parse`/`Marshal`; a +`Document` edits a file in place instead. + +## Editing a document in place + +```go +doc, err := toml.ParseDocument(data) +err = doc.Decode(&settings) +unknown := doc.Unmapped(&settings) // []toml.Key no field takes +removed, err := doc.Delete(toml.Key{"play", "effects"}) +err = doc.Set(toml.Key{"play", "sfx"}, false) +data = doc.Bytes() +``` + +`ParseDocument` parses as `Parse` does and keeps the bytes. An edit rewrites only +the statements it names, so comments, blank lines, order and the spelling of +every other value stay as written; the result is parsed again, and an edit it +would not parse after is refused with the document unchanged. + +- `Set` replaces the value of a key the document sets, keeping the rest of its + line and its comment. A key it lacks goes after the last key of its table, + spelled as that key is (`play.mouse` beside `play.hud`), else after the table's + header, else in a `[table]` appended at the end. A key under an array of + tables, inside an inline table or array, or naming a table fails. The value is + encoded as `Marshal` writes one. +- `Delete` removes the key and every key under it: each statement setting one and + each header naming one, in every table of an array of tables. It reports + whether it removed anything; a key inside an inline value fails. +- `Unmapped` names the keys no field of the destination takes, matched as `Decode` + matches them (tags, then field names; unexported and `-` fields take nothing), + each once and sorted. A map or interface destination takes every key under it; + a key under an array of tables is named without its index. +- `Key` is a dotted path, one segment per part; `Key.String` spells it as a + document would, quoting segments that need it. + +A token's `Pos` and `End` are its byte offsets in the input. ## Deliberate subset and deviations diff --git a/document.go b/document.go new file mode 100644 index 0000000..459b259 --- /dev/null +++ b/document.go @@ -0,0 +1,305 @@ +package toml + +import ( + "bytes" + "cmp" + "fmt" + "reflect" + "slices" + "strings" +) + +// Key is a dotted key path, one segment per part: [a."b.c"] is Key{"a", "b.c"} +type Key []string + +// String spells the key as a document writes it, quoting segments that need it +func (k Key) String() string { + parts := make([]string, len(k)) + for i, s := range k { + if t, err := keyText(s); err == nil { + parts[i] = t + } else { + var e encoder + e.string(s) + parts[i] = e.buf.String() + } + } + return strings.Join(parts, ".") +} + +// under reports k is p or a key inside it +func (k Key) under(p Key) bool { return len(k) >= len(p) && slices.Equal(k[:len(p)], p) } + +// Document is a parsed document that keeps its bytes. An edit rewrites only the +// statements it names, so comments, blank lines, order and the spelling of every +// other value stay as written, and the result is parsed again before it stands. +type Document struct { + data []byte + tree map[string]any + stmts []statement +} + +// statement is a table header or a key and its value, as the lines it spans: from +// the start of its line through its newline, a comment after it on that line +// included. A key is its path relative to scope, the header it sits under. +type statement struct { + start, end int + scope, key Key + header bool + array bool // an [[array]] header + value [2]int // a key's value bytes +} + +func (s statement) path() Key { + if s.header { + return s.key + } + return append(slices.Clip(s.scope), s.key...) +} + +// ParseDocument parses data as Parse does and keeps it for editing; an edit +// writes a new slice, never into data +func ParseDocument(data []byte) (*Document, error) { + tree, err := NewParser(data).Parse() + if err != nil { + return nil, err + } + return &Document{data: data, tree: tree, stmts: statements(data)}, nil +} + +// statements splits a document the parser accepted into its statements +func statements(data []byte) []statement { + lx := NewLexer(data) + t := lx.NextToken() + key := func() Key { + var k Key + for { + k = append(k, t.Literal) + if t = lx.NextToken(); t.Type != TokenDot { + return k + } + t = lx.NextToken() + } + } + // lineEnd takes a trailing comment and the newline, leaving t past them + lineEnd := func() int { + if t.Type == TokenComment { + t = lx.NextToken() + } + end := t.End + if t.Type != TokenEOF { + t = lx.NextToken() + } + return end + } + var ( + out []statement + scope Key + ) + for t.Type != TokenEOF { + s := statement{start: bytes.LastIndexByte(data[:t.Pos], '\n') + 1, scope: scope} + switch t.Type { + case TokenLBracket: + s.header, t = true, lx.NextToken() + if t.Type == TokenLBracket { + s.array, t = true, lx.NextToken() + } + s.key = key() + for t.Type == TokenRBracket { + t = lx.NextToken() + } + s.scope, scope = nil, s.key + case TokenIdent, TokenString: + s.key = key() + t = lx.NextToken() // past = + s.value[0] = t.Pos + for depth := 0; ; { + switch t.Type { + case TokenLBracket, TokenLBrace: + depth++ + case TokenRBracket, TokenRBrace: + depth-- + } + if t.Type != TokenNewline && t.Type != TokenComment { + s.value[1] = t.End + } + if t = lx.NextToken(); depth == 0 && (t.Type == TokenNewline || t.Type == TokenComment || t.Type == TokenEOF) { + break + } + } + default: // a blank or comment line + t = lx.NextToken() + continue + } + s.end = lineEnd() + out = append(out, s) + } + return out +} + +// Bytes is the document as it now reads; the caller must not change it +func (d *Document) Bytes() []byte { return d.data } + +// Tree is the document's values as Parse returns them, which Decode reads +func (d *Document) Tree() map[string]any { return d.tree } + +// Decode stores the document's values in v as Decode does +func (d *Document) Decode(v any) error { return Decode(d.tree, v) } + +// Unmapped names the keys of the document that no field of v's type takes, as +// Decode, which ignores them, would match them; each once and sorted. A key under +// a map or interface destination is taken, and one under an array of tables is +// named without its index. +func (d *Document) Unmapped(v any) []Key { + seen := make(map[string]Key) + unmapped(d.tree, reflect.TypeOf(v), nil, seen, 0) + out := make([]Key, 0, len(seen)) + for _, k := range seen { + out = append(out, k) + } + slices.SortFunc(out, func(a, b Key) int { return slices.Compare(a, b) }) + return out +} + +func unmapped(data any, t reflect.Type, at Key, seen map[string]Key, depth int) { + for t != nil && t.Kind() == reflect.Pointer { + t = t.Elem() + } + if t == nil || depth > maxValueDepth { + return + } + switch t.Kind() { + case reflect.Struct: + m, ok := data.(map[string]any) + if !ok { + return + } + fields := make(map[string]reflect.Type, t.NumField()) + for i := range t.NumField() { + f := t.Field(i) + name, _, _ := strings.Cut(f.Tag.Get("toml"), ",") + if f.IsExported() && name != "-" { + fields[cmp.Or(name, f.Name)] = f.Type + } + } + for k, v := range m { + key := append(slices.Clip(at), k) + if ft, ok := fields[k]; ok { + unmapped(v, ft, key, seen, depth+1) + } else { + seen[key.String()] = key + } + } + case reflect.Map: + if m, ok := data.(map[string]any); ok { + for k, v := range m { + unmapped(v, t.Elem(), append(slices.Clip(at), k), seen, depth+1) + } + } + case reflect.Slice, reflect.Array: + switch s := data.(type) { + case []any: + for _, v := range s { + unmapped(v, t.Elem(), at, seen, depth+1) + } + case []map[string]any: + for _, v := range s { + unmapped(v, t.Elem(), at, seen, depth+1) + } + } + } +} + +// Delete removes key and every key under it: each statement setting one and each +// table header naming one, in every table of an array of tables, every other line +// kept. It reports whether anything was removed; a key inside an inline value, a +// table or array written as a value, cannot be edited and fails. +func (d *Document) Delete(key Key) (bool, error) { + var out []byte + at := 0 + for _, s := range d.stmts { + p := s.path() + if !s.header && len(p) < len(key) && key.under(p) && d.inline(s) { + return false, fmt.Errorf("key %s is inside the value of %s", key, p) + } + if p.under(key) { + out, at = append(out, d.data[at:s.start]...), s.end + } + } + if at == 0 { + return false, nil + } + return true, d.replace(append(out, d.data[at:]...)) +} + +// inline reports a key's value is an inline table or an array +func (d *Document) inline(s statement) bool { + c := d.data[s.value[0]] + return c == '{' || c == '[' +} + +// Set gives key the value v, encoded as Marshal writes a value. A key the document +// sets has its value replaced, the rest of its line kept; a key it lacks goes after +// the last key of its table, else after the table's header, else in a table +// appended to the document. A key under an array of tables or inside an inline +// value fails, and so does a value the document then would not parse with. +func (d *Document) Set(key Key, v any) error { + if len(key) == 0 || v == nil { + return fmt.Errorf("a key and a value are needed") + } + var e encoder + if err := e.value(reflect.ValueOf(v), 0); err != nil { + return fmt.Errorf("key %s: %w", key, err) + } + value := e.buf.Bytes() + parent := key[:len(key)-1] + after, scope := -1, Key(nil) // where a missing key goes, and the scope it is spelled in + for _, s := range d.stmts { + p := s.path() + switch { + case s.header && s.array && key.under(p): + return fmt.Errorf("key %s is under the array of tables %s", key, p) + case !s.header && slices.Equal(p, key): + return d.replace(slices.Concat(d.data[:s.value[0]], value, d.data[s.value[1]:])) + case !s.header && len(p) < len(key) && key.under(p): + return fmt.Errorf("key %s is inside the value of %s", key, p) + case p.under(key): + return fmt.Errorf("key %s is a table", key) + case s.header && slices.Equal(p, parent): + after, scope = s.end, p + case !s.header && len(p) > len(parent) && p.under(parent) && parent.under(s.scope): + if !slices.Equal(scope, parent) || slices.Equal(s.scope, parent) { + after, scope = s.end, s.scope + } + } + } + head := "" + switch { + case after >= 0: + case len(parent) == 0: // a root key goes before the first header + after = len(d.data) + if i := slices.IndexFunc(d.stmts, func(s statement) bool { return s.header }); i >= 0 { + after = d.stmts[i].start + } + default: + after, scope, head = len(d.data), parent, "["+parent.String()+"]\n" + if after > 0 { + head = "\n" + head + } + } + line := slices.Concat([]byte(head+key[len(scope):].String()+" = "), value, []byte("\n")) + if after > 0 && d.data[after-1] != '\n' { + line = slices.Concat([]byte("\n"), line) + } + return d.replace(slices.Concat(d.data[:after], line, d.data[after:])) +} + +// replace makes data the document once it parses +func (d *Document) replace(data []byte) error { + next, err := ParseDocument(data) + if err != nil { + return fmt.Errorf("edit refused: %w", err) + } + *d = *next + return nil +} diff --git a/document_test.go b/document_test.go new file mode 100644 index 0000000..040d23a --- /dev/null +++ b/document_test.go @@ -0,0 +1,268 @@ +package toml + +import ( + "fmt" + "slices" + "testing" +) + +const settingsDoc = `# mine +play.hud = true # root dotted + +[paths] +log = "l" # beside the file + +[play] +# kept +mouse = false # keys +volume = 0.3 + +# about later +[later] # a newer build's +key = 1 +[later.sub] +deep = [ + 1, # one + 2, +] + +[audio] +buffer_ms = 20 +` + +// TestDocumentEditsOnlyWhatItNames: a set value keeps the rest of its line, a +// missing key goes after its table's last key in that key's spelling, a missing +// table is appended, a delete takes a key's lines or a table's header with every +// key under it, and every other byte, comments and blank lines, stays. +func TestDocumentEditsOnlyWhatItNames(t *testing.T) { + d, err := ParseDocument([]byte(settingsDoc)) + if err != nil { + t.Fatal(err) + } + for _, edit := range []func() error{ + func() error { return d.Set(Key{"paths", "log"}, "logs") }, + func() error { return d.Set(Key{"play", "sfx"}, false) }, + func() error { return d.Set(Key{"presentation", "cell"}, 14.0) }, + func() error { return d.Set(Key{"audio", "buffer ms"}, 30) }, + func() error { return del(d, Key{"play", "volume"}) }, + func() error { return del(d, Key{"later"}) }, + } { + if err := edit(); err != nil { + t.Fatal(err) + } + } + want := `# mine +play.hud = true # root dotted + +[paths] +log = "logs" # beside the file + +[play] +# kept +mouse = false # keys +sfx = false + +# about later + +[audio] +buffer_ms = 20 +"buffer ms" = 30 + +[presentation] +cell = 14.0 +` + if got := string(d.Bytes()); got != want { + t.Fatalf("edited:\n%s\nwant:\n%s", got, want) + } + var s struct { + Play struct { + HUD bool `toml:"hud"` + Mouse bool `toml:"mouse"` + Sfx bool `toml:"sfx"` + } `toml:"play"` + } + s.Play.Sfx = true + if err := d.Decode(&s); err != nil || !s.Play.HUD || s.Play.Mouse || s.Play.Sfx { + t.Errorf("decoded %+v, %v", s, err) + } +} + +func del(d *Document, k Key) error { + ok, err := d.Delete(k) + if err == nil && !ok { + err = fmt.Errorf("%s: nothing deleted", k) + } + return err +} + +// TestSetSpellsAKeyAsItsTableDoes: with no header, a key joins the dotted keys that +// made its table, spelled as they are. +func TestSetSpellsAKeyAsItsTableDoes(t *testing.T) { + d, err := ParseDocument([]byte("play.hud = true\n\n[audio]\nbuffer_ms = 20\n")) + if err != nil { + t.Fatal(err) + } + if err := d.Set(Key{"play", "mouse"}, false); err != nil { + t.Fatal(err) + } + if got, want := string(d.Bytes()), "play.hud = true\nplay.mouse = false\n\n[audio]\nbuffer_ms = 20\n"; got != want { + t.Fatalf("got:\n%s\nwant:\n%s", got, want) + } +} + +// TestDocumentRefusesAnEditItCannotPlace: under an array of tables, inside an +// inline value, or over a table, an edit fails and the document stays as it was. +func TestDocumentRefusesAnEditItCannotPlace(t *testing.T) { + doc := "x = 1\nm = { a = 1 }\n[t]\nk = 1\n[[arr]]\nn = 1\n" + d, err := ParseDocument([]byte(doc)) + if err != nil { + t.Fatal(err) + } + for name, edit := range map[string]func() error{ + "array of tables": func() error { return d.Set(Key{"arr", "n"}, 2) }, + "inline set": func() error { return d.Set(Key{"m", "b"}, 2) }, + "inline delete": func() error { _, err := d.Delete(Key{"m", "a"}); return err }, + "table": func() error { return d.Set(Key{"t"}, 2) }, + "under a value": func() error { return d.Set(Key{"x", "y"}, 2) }, + } { + if err := edit(); err == nil { + t.Errorf("%s: the edit was taken", name) + } + } + if string(d.Bytes()) != doc { + t.Fatalf("a refused edit changed the document:\n%s", d.Bytes()) + } +} + +// TestUnmappedNamesWhatDecodeIgnores: keys no field takes, by tag or field name, +// unexported and "-" fields included, are named once each, under arrays of tables +// without their index; a map or interface destination takes every key under it. +func TestUnmappedNamesWhatDecodeIgnores(t *testing.T) { + doc := `top = 1 +skip = 2 +hidden = 3 +Named = 4 +[sub] +ok = 1 +bad = 2 +[sub.deeper] +x = 1 +[free] +anything = 1 +[anyv] +v = { w = 1 } +[[items]] +id = 1 +odd = 1 +[[items]] +id = 2 +odd = 2 +[byname.one] +ok = 1 +nope = 1 +` + type item struct { + ID int `toml:"id"` + } + type sub struct { + OK int `toml:"ok,omitempty"` + } + var v struct { + Top int `toml:"top"` + Skip int `toml:"-"` + hidden int + Named int `toml:",omitempty"` + Sub *sub `toml:"sub"` + Free map[string]any `toml:"free"` + Anyv any `toml:"anyv"` + Items []item `toml:"items"` + ByName map[string]sub `toml:"byname"` + } + d, err := ParseDocument([]byte(doc)) + if err != nil { + t.Fatal(err) + } + var got []string + for _, k := range d.Unmapped(&v) { + got = append(got, k.String()) + } + want := []string{"byname.one.nope", "hidden", "items.odd", "skip", "sub.bad", "sub.deeper"} + if !slices.Equal(got, want) { + t.Fatalf("unmapped %q, want %q", got, want) + } +} + +// TestKeyStringQuotesWhatABareKeyCannotHold +func TestKeyStringQuotesWhatABareKeyCannotHold(t *testing.T) { + if got := (Key{"a", "b.c", "d e", "7"}).String(); got != `a."b.c"."d e"."7"` { + t.Fatalf("got %s", got) + } +} + +// FuzzDocument: a document that parses keeps its bytes; deleting any key it sets +// leaves a shorter document that parses without that key, and setting one or a +// key beside it, where the edit is taken, leaves it holding the value set. +func FuzzDocument(f *testing.F) { + for _, s := range []string{ + settingsDoc, + "a = 1\n[t]\nk = [1,\n2] # c\n[[r]]\nx = 1\n[[r]]\nx = 2\n", + "t.a.b = { c = 1 }\n# end", + "\"q.k\" = 1\r\n[\"x y\".z]\r\nv = \"#\"\r\n", + } { + f.Add([]byte(s)) + } + f.Fuzz(func(t *testing.T, data []byte) { + d, err := ParseDocument(data) + if err != nil { + return + } + if string(d.Bytes()) != string(data) { + t.Fatal("a parsed document changed its bytes") + } + for _, s := range d.stmts { + c, _ := ParseDocument(data) + ok, err := c.Delete(s.path()) + if err != nil { + continue + } + if !ok { + t.Fatalf("delete of %s, which the document sets, removed nothing", s.path()) + } + if has(c.Tree(), s.path()) { + t.Fatalf("%s still set after its delete:\n%s", s.path(), c.Bytes()) + } + if len(c.Bytes()) >= len(data) { + t.Fatalf("delete of %s did not shorten the document", s.path()) + } + } + for _, s := range d.stmts { + for _, k := range []Key{s.path(), append(s.path()[:len(s.path())-1:len(s.path())-1], "added")} { + c, _ := ParseDocument(data) + if err := c.Set(k, int64(7)); err == nil && value(c.Tree(), k) != int64(7) { + t.Fatalf("set %s took but reads %v:\n%s", k, value(c.Tree(), k), c.Bytes()) + } + } + } + }) +} + +func has(tree map[string]any, k Key) bool { return value(tree, k) != nil } + +// value is what k holds in tree, in the last table of an array of tables +func value(tree map[string]any, k Key) any { + var cur any = tree + for _, s := range k { + m, ok := cur.(map[string]any) + if !ok { + if a, ok := cur.([]map[string]any); ok && len(a) > 0 { + m = a[len(a)-1] + } else { + return nil + } + } + if cur, ok = m[s]; !ok { + return nil + } + } + return cur +} diff --git a/lexer.go b/lexer.go index 59b5feb..03484ba 100644 --- a/lexer.go +++ b/lexer.go @@ -23,7 +23,13 @@ func NewLexer(input []byte) *Lexer { func (l *Lexer) NextToken() Token { l.skipWhitespace() + start := l.pos + t := l.next() + t.Pos, t.End = start, l.pos + return t +} +func (l *Lexer) next() Token { if l.pos >= len(l.input) { return l.newToken(TokenEOF, "") } diff --git a/token.go b/token.go index 8fb83fa..fb65d0f 100644 --- a/token.go +++ b/token.go @@ -30,12 +30,13 @@ const ( TokenNewline // \n ) -// Token represents a lexical token +// Token represents a lexical token; Pos and End are its byte offsets in the input type Token struct { - Type TokenType - Literal string - Line int - Col int + Type TokenType + Literal string + Line int + Col int + Pos, End int } func (t Token) String() string {