From 4dbb86e0d222c2c43cb8867a8e196cde46e9e77d Mon Sep 17 00:00:00 2001 From: Tiago Natel de Moura Date: Wed, 29 Mar 2017 01:57:47 -0300 Subject: [PATCH 1/3] Enables | close to variable token Signed-off-by: Tiago Natel de Moura --- scanner/lex.go | 16 ++++++++++- scanner/lex_regression_test.go | 9 +++++++ scanner/lex_test.go | 49 ++++++++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 1 deletion(-) diff --git a/scanner/lex.go b/scanner/lex.go index 564df6bb..781df447 100644 --- a/scanner/lex.go +++ b/scanner/lex.go @@ -282,11 +282,25 @@ func lexStart(l *Lexer) stateFn { absorbIdentifier(l) + // required to avoid the sintaxes below: + // rm -rf $HOME/projects + // rm -rf $GOPATH/test + // for being interpreted as: + // rm -rf $HOME /projects + // rm -rf $GOPATH /test + // + // The list of runes below are the ones allowed to exists close + // to the variable. Eg.: + // echo $HOME; + // echo $HOME[0] + // echo $HOME() + // echo $HOME+"a" next := l.peek() if next != eof && !isSpace(next) && !isEndOfLine(next) && next != ';' && next != ')' && next != ',' && next != '+' && - next != '[' && next != ']' && next != '(' { + next != '[' && next != ']' && next != '(' && + next != '|' { l.errorf("Unrecognized character in action: %#U", next) return nil } diff --git a/scanner/lex_regression_test.go b/scanner/lex_regression_test.go index eec53de4..33e65376 100644 --- a/scanner/lex_regression_test.go +++ b/scanner/lex_regression_test.go @@ -207,4 +207,13 @@ func TestLexerIssue127(t *testing.T) { } testTable("test127", `rm -rf $HOME/.vim`, expected, t) + + expected = []Token{ + {typ: token.Ident, val: "rm"}, + {typ: token.Arg, val: "-rf"}, + {typ: token.Illegal, val: "test127:1:12: Unrecognized character in action: U+002E '.'"}, + {typ: token.EOF}, + } + + testTable("test127", `rm -rf $HOME.vim`, expected, t) } diff --git a/scanner/lex_test.go b/scanner/lex_test.go index 6897cc0c..669afa5c 100644 --- a/scanner/lex_test.go +++ b/scanner/lex_test.go @@ -1827,3 +1827,52 @@ func TestLexerLongAssignment(t *testing.T) { jq ".GroupId" | xargs echo -n)`, expected, t) } + +func TestLexerVariable(t *testing.T) { + base := []Token{ + {typ: token.Variable, val: "$HOME"}, + } + + for r, tok := range map[rune]token.Token{ + ';': token.Semicolon, + '+': token.Plus, + '[': token.LBrack, + ']': token.RBrack, + '(': token.LParen, + ')': token.RParen, + ',': token.Comma, + '|': token.Pipe, + } { + expected := append(base, Token{typ: tok, val: string(r)}) + + if r == ')' { + expected = append(expected, Token{typ: token.Semicolon, val: ";"}) + } + + expected = append(expected, Token{typ: token.EOF}) + testTable("test variable", `$HOME`+string(r), expected, t) + } + + // must fail + for _, r := range []rune{ + '!', '@', '$', '%', '&', '*', '-', '=', '"', '\'', + '`', '{', '}', '<', '>', '.', ':', '?', '/', '\\', + } { + l := Lex("test", "$GOPATH"+string(r)) + + var results []Token + + for tok := range l.Tokens { + results = append(results, tok) + } + + if len(results) != 2 { + t.Fatalf("Expected 2 token, found %d", len(results)) + } + + if results[0].typ != token.Illegal { + t.Fatalf("Expected illegal token, but found %v", results[0]) + } + } + +} From 3eabf85fbcbdf977e8634b75478a98388e5a10c5 Mon Sep 17 00:00:00 2001 From: Tiago Natel de Moura Date: Wed, 29 Mar 2017 02:00:05 -0300 Subject: [PATCH 2/3] fix comment Signed-off-by: Tiago Natel de Moura --- scanner/lex.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scanner/lex.go b/scanner/lex.go index 781df447..808ecd44 100644 --- a/scanner/lex.go +++ b/scanner/lex.go @@ -291,10 +291,10 @@ func lexStart(l *Lexer) stateFn { // // The list of runes below are the ones allowed to exists close // to the variable. Eg.: - // echo $HOME; - // echo $HOME[0] - // echo $HOME() - // echo $HOME+"a" + // $HOME; + // $HOME[0] + // $HOME() + // $HOME+"a" next := l.peek() if next != eof && !isSpace(next) && !isEndOfLine(next) && next != ';' && From aab08e2779f3f615758fbf52af79057a9607eabb Mon Sep 17 00:00:00 2001 From: Tiago Natel de Moura Date: Mon, 7 Aug 2017 17:54:48 -0300 Subject: [PATCH 3/3] improve the code and comments Signed-off-by: Tiago Natel de Moura --- scanner/lex.go | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/scanner/lex.go b/scanner/lex.go index 808ecd44..4e698e13 100644 --- a/scanner/lex.go +++ b/scanner/lex.go @@ -282,6 +282,8 @@ func lexStart(l *Lexer) stateFn { absorbIdentifier(l) + next := l.peek() + // required to avoid the sintaxes below: // rm -rf $HOME/projects // rm -rf $GOPATH/test @@ -289,18 +291,14 @@ func lexStart(l *Lexer) stateFn { // rm -rf $HOME /projects // rm -rf $GOPATH /test // - // The list of runes below are the ones allowed to exists close - // to the variable. Eg.: + // Below are the valid suffixes for variables: // $HOME; - // $HOME[0] - // $HOME() - // $HOME+"a" - next := l.peek() - if next != eof && !isSpace(next) && - !isEndOfLine(next) && next != ';' && - next != ')' && next != ',' && next != '+' && - next != '[' && next != ']' && next != '(' && - next != '|' { + // $HOME[ # used in: $HOME[0] + // $HOME( # used in: $callback() + // $HOME+ # used in: $HOME+"/src" + // $HOME] # used in: $list[$index] + // $HOME) # used in: call($HOME) + if !isValidVariableSuffix(next) { l.errorf("Unrecognized character in action: %#U", next) return nil } @@ -564,3 +562,11 @@ func isAlpha(r rune) bool { func isEndOfLine(r rune) bool { return r == '\r' || r == '\n' } + +func isValidVariableSuffix(r rune) bool { + return r == eof || isSpace(r) || + isEndOfLine(r) || r == ';' || + r == ')' || r == ',' || r == '+' || + r == '[' || r == ']' || r == '(' || + r == '|' +}