From 1898c8a434372fbf789e7dbf58648e938922c4db Mon Sep 17 00:00:00 2001
From: blueloveTH <blueloveTH@foxmail.com>
Date: Mon, 5 Jun 2023 18:30:21 +0800
Subject: [PATCH] fix a lexer bug

---
 src/lexer.h      | 44 +++++++++++++++++++++++---------------------
 tests/99_bugs.py | 13 ++++++++++++-
 2 files changed, 35 insertions(+), 22 deletions(-)

diff --git a/src/lexer.h b/src/lexer.h
index c3f09fd6..e8d9f945 100644
--- a/src/lexer.h
+++ b/src/lexer.h
@@ -231,25 +231,6 @@ struct Lexer {
         }
 
         if(kTokenKwMap.count(name)){
-            if(name == "not"){
-                if(strncmp(curr_char, " in", 3) == 0){
-                    curr_char += 3;
-                    add_token(TK("not in"));
-                    return 0;
-                }
-            }else if(name == "is"){
-                if(strncmp(curr_char, " not", 4) == 0){
-                    curr_char += 4;
-                    add_token(TK("is not"));
-                    return 0;
-                }
-            }else if(name == "yield"){
-                if(strncmp(curr_char, " from", 5) == 0){
-                    curr_char += 5;
-                    add_token(TK("yield from"));
-                    return 0;
-                }
-            }
             add_token(kTokenKwMap.at(name));
         } else {
             add_token(TK("@id"));
@@ -276,13 +257,34 @@ struct Lexer {
             case TK("{"): case TK("["): case TK("("): brackets_level++; break;
             case TK(")"): case TK("]"): case TK("}"): brackets_level--; break;
         }
-        nexts.push_back( Token{
+        auto token = Token{
             type,
             token_start,
             (int)(curr_char - token_start),
             current_line - ((type == TK("@eol")) ? 1 : 0),
             value
-        });
+        };
+        // handle "not in", "is not", "yield from"
+        if(!nexts.empty()){
+            switch(nexts.back().type){
+                case TK("not"):
+                    if(type == TK("in")) {
+                        nexts.back().type = TK("not in");
+                        return;
+                    }
+                case TK("is"):
+                    if(type == TK("not")){
+                        nexts.back().type = TK("is not");
+                        return;
+                    }
+                case TK("yield"):
+                    if(type == TK("from")){
+                        nexts.back().type = TK("yield from");
+                        return;
+                    }
+                default: nexts.push_back(token); return;
+            }
+        }
     }
 
     void add_token_2(char c, TokenIndex one, TokenIndex two) {
diff --git a/tests/99_bugs.py b/tests/99_bugs.py
index 4eeb827a..f08f4647 100644
--- a/tests/99_bugs.py
+++ b/tests/99_bugs.py
@@ -18,4 +18,15 @@ b = [3, 4]
 assert a.append == a.append
 assert a.append is not a.append
 assert a.append is not b.append
-assert a.append != b.append
\ No newline at end of file
+assert a.append != b.append
+
+inq = 0
+if not inq:
+    assert True
+else:
+    assert False
+
+if inq is   not 1:
+    assert True
+if inq  is  not  0:
+    assert False
\ No newline at end of file