From 72e4046137989d98612e4d76599c056cdd8080e8 Mon Sep 17 00:00:00 2001 From: Manuele Conti Date: Sat, 10 Apr 2021 15:50:55 +0200 Subject: [PATCH] Add IGNORECASE ability to Token Remapping. With reference to: https://github.com/dabeaz/sly/issues/62 I've found a problem with tokens remapping in the eventuality of: reflags = re.IGNORECASE This change allows creating special cases ignoring capitalization in the eventuality of reflags = re.IGNORECASE assignment in Lexer class. Example: class MyLexer(sly.Lexer): reflags = re.IGNORECASE # Base ID rule ID = r'[a-zA-Z_][a-zA-Z0-9_]*' # Special cases ID['if'] = IF ID['else'] = ELSE ID['while'] = WHILE # Now if,If,IF,iF are handle as the same special cases --- sly/lex.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/sly/lex.py b/sly/lex.py index 2f3a345..f383c86 100644 --- a/sly/lex.py +++ b/sly/lex.py @@ -277,6 +277,8 @@ def _build(cls): for (key, val), newtok in cls._remap.items(): if key not in cls._remapping: cls._remapping[key] = {} + if cls.reflags & re.IGNORECASE: + val = val.lower() cls._remapping[key][val] = newtok remapped_toks = set() @@ -411,7 +413,11 @@ def _reject(): tok.type = m.lastgroup if tok.type in _remapping: - tok.type = _remapping[tok.type].get(tok.value, tok.type) + if self.reflags & re.IGNORECASE: + token_search = tok.value.lower() + else: + token_search = tok.value + tok.type = _remapping[tok.type].get(token_search, tok.type) if tok.type in _token_funcs: self.index = index