Skip to content

Commit c8e65cd

Browse files
niliumnilium
committed
Add support for ending statements on newline
This adds a bitflag to the parser to enable parsing newlines as statement sentinels in place of semicolons. Semicolons continue to work. This still needs work to support a backslash before EOL as a means of ending a statement, but since there's no lookahead on tokens right now, that's a little tricky. Might be doable by checking if the last element of the current statement is a '\' word.
1 parent f209164 commit c8e65cd

File tree

2 files changed

+115
-17
lines changed

2 files changed

+115
-17
lines changed

‎parser.go‎

Lines changed: 56 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,19 @@ type TokenReader interface {
1515
ReadToken() (Token, error)
1616
}
1717

18+
// ParseFlags is a bitset of any boolean flags that can be set for a Parser.
19+
type ParseFlags uint
20+
21+
func (f ParseFlags) isSet(flags ParseFlags) bool {
22+
return f&flags == flags
23+
}
24+
25+
const (
26+
// Whether to treat an end-of-line (newline) as a sentinel. This is the same as using
27+
// newlines instead of semicolons.
28+
ParseSentinelEOL ParseFlags = 1 << iota
29+
)
30+
1831
// Parser consumes tokens from a TokenReader and constructs a codf *Document from it.
1932
//
2033
// The Document produced by the Parser is kept for the duration of the parser's lifetime, so it is
@@ -23,6 +36,8 @@ type Parser struct {
2336
doc *Document
2437
next tokenConsumer
2538

39+
flags ParseFlags
40+
2641
lastToken Token
2742
lastErr error
2843

@@ -49,6 +64,16 @@ func NewParser() *Parser {
4964
return p
5065
}
5166

67+
// Flags returns the current ParseFlags for the receiver.
68+
func (p *Parser) Flags() ParseFlags {
69+
return p.flags
70+
}
71+
72+
// SetFlags sets the ParseFlags for the receiver.
73+
func (p *Parser) SetFlags(flags ParseFlags) {
74+
p.flags = flags
75+
}
76+
5277
func (p *Parser) nextToken(tr TokenReader) (tok Token, err error) {
5378
tok, err = tr.ReadToken()
5479
p.lastToken, p.lastErr = tok, err
@@ -107,7 +132,7 @@ func (p *Parser) ParseExpr(tr TokenReader) (ExprNode, error) {
107132
exp := exprParser{}
108133
p.ctx = []parseNode{&exp}
109134
p.parseErr = nil
110-
p.next = skipWhitespace(p.parseStatement)
135+
p.next = p.skipInsignificantWhitespace(p.parseStatement)
111136
if err := p.Parse(tr); err != nil {
112137
return nil, err
113138
}
@@ -220,7 +245,7 @@ func (p *Parser) beginSegment(tok Token) (tokenConsumer, error) {
220245
// Start statement
221246
stmt := &Statement{NameTok: &Literal{tok}}
222247
p.pushContext(stmt)
223-
return skipWhitespace(p.parseStatement), nil
248+
return p.skipInsignificantWhitespace(p.parseStatement), nil
224249
}
225250
return nil, unexpected(tok, "expected statement or section name")
226251
}
@@ -236,11 +261,35 @@ func skipWhitespace(next tokenConsumer) (consumer tokenConsumer) {
236261
return consumer
237262
}
238263

264+
func (p *Parser) skipInsignificantWhitespace(next tokenConsumer) (consumer tokenConsumer) {
265+
if !p.flags.isSet(ParseSentinelEOL) {
266+
return skipWhitespace(next)
267+
}
268+
consumer = func(tok Token) (tokenConsumer, error) {
269+
if tok.Kind == TWhitespace && tok.Start.Line == tok.End.Line {
270+
return consumer, nil
271+
} else if tok.Kind == TComment {
272+
return consumer, nil
273+
}
274+
return next(tok)
275+
}
276+
return consumer
277+
}
278+
239279
func (p *Parser) parseStatementSentinel(tok Token) (tokenConsumer, error) {
240280
switch tok.Kind {
241281
case TEOF:
242282
return nil, p.closeError(tok)
243283

284+
case TWhitespace:
285+
if stmt, ok := p.context().(*Statement); ok {
286+
p.popContext()
287+
stmt.EndTok = tok
288+
p.context().(parentNode).addChild(stmt)
289+
return p.beginSegment, nil
290+
}
291+
return nil, p.closeError(tok)
292+
244293
case TSemicolon:
245294
if stmt, ok := p.context().(*Statement); ok {
246295
p.popContext()
@@ -257,7 +306,7 @@ func (p *Parser) parseStatementSentinel(tok Token) (tokenConsumer, error) {
257306
if err := p.context().(segmentNode).addExpr(ary); err != nil {
258307
return nil, err
259308
}
260-
return skipWhitespace(p.parseStatement), nil
309+
return p.skipInsignificantWhitespace(p.parseStatement), nil
261310
}
262311
return nil, p.closeError(tok)
263312

@@ -272,7 +321,7 @@ func (p *Parser) parseStatementSentinel(tok Token) (tokenConsumer, error) {
272321
if err := p.context().(segmentNode).addExpr(m); err != nil {
273322
return nil, err
274323
}
275-
return skipWhitespace(p.parseStatement), nil
324+
return p.skipInsignificantWhitespace(p.parseStatement), nil
276325
}
277326
return nil, p.closeError(tok)
278327

@@ -294,14 +343,14 @@ func (p *Parser) beginArray(tok Token) (tokenConsumer, error) {
294343
StartTok: tok,
295344
Elems: []ExprNode{},
296345
})
297-
return skipWhitespace(p.parseStatement), nil
346+
return p.skipInsignificantWhitespace(p.parseStatement), nil
298347
}
299348

300349
func (p *Parser) beginMap(tok Token) (tokenConsumer, error) {
301350
m := newMapBuilder()
302351
m.m.StartTok = tok
303352
p.pushContext(m)
304-
return skipWhitespace(p.parseStatement), nil
353+
return p.skipInsignificantWhitespace(p.parseStatement), nil
305354
}
306355

307356
func (p *Parser) parseStatement(tok Token) (tokenConsumer, error) {
@@ -327,7 +376,7 @@ func (p *Parser) parseStatement(tok Token) (tokenConsumer, error) {
327376
if err := p.context().(segmentNode).addExpr(&Literal{tok}); err != nil {
328377
return nil, err
329378
}
330-
return skipWhitespace(p.parseStatement), nil
379+
return p.skipInsignificantWhitespace(p.parseStatement), nil
331380
}
332381

333382
return p.parseStatementSentinel(tok)

‎parser_test.go‎

Lines changed: 59 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -7,32 +7,44 @@ import (
77
"time"
88
)
99

10-
func parse(in string) (*Document, error) {
10+
type configFunc func(*Lexer, *Parser)
11+
12+
func parserSetSentinelEOL(_ *Lexer, p *Parser) {
13+
p.SetFlags(ParseSentinelEOL)
14+
if p.Flags()&ParseSentinelEOL != ParseSentinelEOL {
15+
panic("unable to set ParseSentinelEOL")
16+
}
17+
}
18+
19+
func parse(in string, config ...configFunc) (*Document, error) {
1120
r := strings.NewReader(in)
1221
l := NewLexer(r)
1322
p := NewParser()
23+
for _, fn := range config {
24+
fn(l, p)
25+
}
1426
if err := p.Parse(l); err != nil {
1527
return nil, err
1628
}
1729
return p.Document(), nil
1830
}
1931

20-
func mustParse(t *testing.T, in string) *Document {
21-
doc, err := parse(in)
32+
func mustParse(t *testing.T, in string, config ...configFunc) *Document {
33+
doc, err := parse(in, config...)
2234
if err != nil {
2335
t.Fatalf("Parse(..) error = %v; want nil", err)
2436
}
2537
t.Logf("-------- DOCUMENT --------\n%s\n------ END DOCUMENT ------", doc)
2638
return doc
2739
}
2840

29-
func mustParseNamed(t *testing.T, name string, in string) *Document {
41+
func mustParseNamed(t *testing.T, name string, in string, config ...configFunc) *Document {
3042
doc := mustParse(t, in)
3143
doc.Name = name
3244
return doc
3345
}
3446

35-
func mustNotParse(t *testing.T, in string) *Document {
47+
func mustNotParse(t *testing.T, in string, config ...configFunc) *Document {
3648
doc, err := parse(in)
3749
if err == nil {
3850
t.Fatalf("Parse(..) error = %v; want error", err)
@@ -43,10 +55,11 @@ func mustNotParse(t *testing.T, in string) *Document {
4355

4456
// parseTestCase is used to describe and run a parser test, optionally as a subtest.
4557
type parseTestCase struct {
46-
Name string
47-
Src string
48-
Doc *Document
49-
Fun func(*testing.T, string) *Document
58+
Name string
59+
Src string
60+
Doc *Document
61+
Fun func(*testing.T, string, ...configFunc) *Document
62+
Config []configFunc
5063
}
5164

5265
func (p parseTestCase) RunSubtest(t *testing.T) {
@@ -59,7 +72,7 @@ func (p parseTestCase) Run(t *testing.T) {
5972
if fn == nil {
6073
fn = mustParse
6174
}
62-
doc := fn(t, p.Src)
75+
doc := fn(t, p.Src, p.Config...)
6376
objectsEqual(t, "", doc, p.Doc)
6477
}
6578

@@ -203,6 +216,42 @@ func TestParseExample(t *testing.T) {
203216
}.Run(t)
204217
}
205218

219+
func TestParseExampleSentinelEOL(t *testing.T) {
220+
const exampleSource = `server go.spiff.io {
221+
// Retain some semicolons to see they're still the same
222+
listen 0.0.0.0:80;
223+
control unix:///var/run/httpd.sock
224+
proxy unix:///var/run/go-redirect.sock {
225+
strip-x-headers yes
226+
log-access no;
227+
}
228+
// keep caches in 64mb of memory
229+
cache memory 64mb {
230+
expire 10m 404
231+
expire 1h 301 302;
232+
expire 5m 200
233+
}
234+
}`
235+
236+
parseTestCase{
237+
Src: exampleSource,
238+
Config: []configFunc{parserSetSentinelEOL},
239+
Doc: doc().
240+
section("server", "go.spiff.io").
241+
/* server */ statement("listen", "0.0.0.0:80").
242+
/* server */ statement("control", "unix:///var/run/httpd.sock").
243+
/* server */ section("proxy", "unix:///var/run/go-redirect.sock").
244+
/* server */ /* proxy */ statement("strip-x-headers", true).
245+
/* server */ /* proxy */ statement("log-access", false).
246+
/* server */ up().
247+
/* server */ section("cache", "memory", "64mb").
248+
/* server */ /* cache */ statement("expire", time.Minute*10, 404).
249+
/* server */ /* cache */ statement("expire", time.Hour, 301, 302).
250+
/* server */ /* cache */ statement("expire", time.Minute*5, 200).
251+
Doc(),
252+
}.Run(t)
253+
}
254+
206255
func TestParseEmpty(t *testing.T) {
207256
t.Run("Empty", func(t *testing.T) {
208257
objectsEqual(t, "",

0 commit comments

Comments
 (0)