Python syntax highlighting

python/main.py

@@ -2,6 +2,7 @@ from scene import Scene
 from particles import Particles
 from triangles import Triangles
 
+# Palette class (inherits from scene)
 class Palette(Scene):
 
     def __init__(self):

python/syntaxTest.py

@@ -0,0 +1,15 @@
+
+#asdlkjasd
+test #this is a test
+
+print("test")
+
+
+     9 . 42 .12 9.1.1 80. 80.0123
+
+    "" asd
+    "
+    "Test"
+
+
+

src/Pycron.h

@@ -31,6 +31,8 @@ public:
     void StartGameLoop();
     void bindMethods();
 
+    static std::string loadFileToString(const std::string& filename);
+
     static pkpy::PyObject* getRandomNumber(pkpy::VM* vm, pkpy::ArgsView args);
     static pkpy::PyObject* getSin(pkpy::VM* vm, pkpy::ArgsView args);
     static pkpy::PyObject* getCos(pkpy::VM* vm, pkpy::ArgsView args);
@@ -39,7 +41,5 @@ public:
     static pkpy::PyObject* getKeyDown(pkpy::VM* vm, pkpy::ArgsView args);
     static pkpy::PyObject* getMousePressed(pkpy::VM* vm, pkpy::ArgsView args);
     static pkpy::PyObject* getMouseDown(pkpy::VM* vm, pkpy::ArgsView args);
-
-    static std::string loadFileToString(const std::string& filename);
 };
 

src/States/Editor/EditorState.cpp

@@ -17,14 +17,24 @@ EditorState::EditorState(pkpy::VM *vm, Graphics *graphics) : m_vm(vm), m_graphic
 
     std::string randomSource = Pycron::loadFileToString("../python/main.py");
 
-    m_baseBackgroundColor = 59;
+    m_baseBackgroundColor = 56;
-    m_baseTextColor = 51;
+    m_baseTextColor = 63;
-    m_lineNumberBackgroundColor = 58;
+    m_lineNumberBackgroundColor = 57;
-    m_lineNumberTextColor  = 61;
+    m_lineNumberTextColor  = 6;
+    m_unknownTextColor  = 4;
+    m_identifierTextColor  = 43;
+    m_keywordTextColor  = 31;
+    m_builtinTextColor  = 23;
+    m_numericalLiteralTextColor = 32;
+    m_stringLiteralTextColor = 6;
+    m_punctuationTextColor = 52;
+    m_operatorTextColor = 45;
+    m_commentTextColor = 60;
 
-    m_topLetterSpacing = 1;
+
+    m_topLetterSpacing = 0;
     m_bottomLetterSpacing = 1;
-    m_leftLetterSpacing = 1;
+    m_leftLetterSpacing = 0;
     m_rightLetterSpacing = 1;
 
     m_charWidth = m_leftLetterSpacing + m_graphics->GetCurrentFontWidth() + m_rightLetterSpacing; // Final size of char with spacing. If the literal width of the font is n, the final width is n + spacing.
@@ -37,13 +47,14 @@ EditorState::EditorState(pkpy::VM *vm, Graphics *graphics) : m_vm(vm), m_graphic
     m_foregroundIndexBuffer = std::vector<uint8_t>(m_textWidth * m_textHeight);
     m_backgroundIndexBuffer = std::vector<uint8_t>(m_textWidth * m_textHeight);
 
+    m_dirtyFlags = std::vector<bool>(m_textHeight);
+
     Clear();
 
-
-    std::string testText = "This is a test\n[Test]\nThis is the end of the test.";
-
     LoadStringToBuffer(randomSource);
 
+    auto tokens = m_pythonTokenizer->tokenizeLine("for i in range(100):");
+
 }
 
 EditorState::~EditorState() {
@@ -52,13 +63,14 @@ EditorState::~EditorState() {
 
 void EditorState::Draw() {
     m_graphics->Clear(0);
-    
+
+    int bg = 28;
 
     for (int i = 0; i < m_characterBuffer.size(); ++i) {
         int x = (i % m_textWidth) * m_charWidth;
         int y = (i / m_textWidth) * m_charHeight;
         m_graphics->Rect(x, y, m_charWidth, m_charHeight, m_backgroundIndexBuffer[i]);
-        m_graphics->Char(m_characterBuffer[i], x + m_leftLetterSpacing + 1, y + m_topLetterSpacing + 1, 0);
+        m_graphics->Char(m_characterBuffer[i], x + m_leftLetterSpacing + 1, y + m_topLetterSpacing + 1, bg);
         m_graphics->Char(m_characterBuffer[i], x + m_leftLetterSpacing, y + m_topLetterSpacing, m_foregroundIndexBuffer[i]);
     }
 
@@ -66,12 +78,42 @@ void EditorState::Draw() {
         Clear();
         m_dirty = false;
         for (int i = 0; i < m_text.size(); ++i) {
+            // Line numbers TODO: maybe not have this as part of the buffer, instead as a custom bar. (Allows for more custom functionality such as bookmarks)
             std::string lineNumber = std::to_string(i);
             int size = 2;
             int diff = size - (int)lineNumber.size();
             if(diff > 0) lineNumber = std::string(diff, ' ') + lineNumber;
             Text(lineNumber, 0, i, m_lineNumberTextColor, m_lineNumberBackgroundColor);
-            Text(m_text[i], size, i, m_baseTextColor, m_baseBackgroundColor);
+
+            // Text handling
+            auto tokens = m_pythonTokenizer->tokenizeLine(m_text[i]);
+
+            int currentPos = 0;
+            for (int j = 0; j < tokens.size(); ++j) {
+                int color = m_baseTextColor;
+                TokenType type = tokens[j].type;
+                if(type == TokenType::Identifier){
+                    color = m_identifierTextColor;
+                }else if(type == TokenType::Keyword){
+                    color = m_keywordTextColor;
+                }else if(type == TokenType::Builtin){
+                    color = m_builtinTextColor;
+                }else if(type == TokenType::NumericalLiteral){
+                    color = m_numericalLiteralTextColor;
+                }else if(type == TokenType::StringLiteral){
+                    color = m_stringLiteralTextColor;
+                }else if(type == TokenType::Punctuation){
+                    color = m_punctuationTextColor;
+                }else if(type == TokenType::Operator){
+                    color = m_operatorTextColor;
+                }else if(type == TokenType::Comment){
+                    color = m_commentTextColor;
+                }else if(type == TokenType::Unknown){
+                    color = m_baseTextColor;
+                }
+                Text(tokens[j].value, size + currentPos, i, color, m_baseBackgroundColor);
+                currentPos += tokens[j].value.size();
+            }
         }
     }
 

src/States/Editor/EditorState.h

@@ -34,12 +34,23 @@ private:
 
     // Text Data
     std::vector<std::string> m_text;
+    std::vector<bool> m_dirtyFlags;
 
     // Theming
     uint8_t m_baseBackgroundColor;
     uint8_t m_baseTextColor;
     uint8_t m_lineNumberBackgroundColor;
     uint8_t m_lineNumberTextColor;
+    uint8_t m_unknownTextColor;
+    uint8_t m_identifierTextColor;
+    uint8_t m_keywordTextColor;
+    uint8_t m_builtinTextColor;
+    uint8_t m_numericalLiteralTextColor;
+    uint8_t m_stringLiteralTextColor;
+    uint8_t m_punctuationTextColor;
+    uint8_t m_operatorTextColor;
+    uint8_t m_commentTextColor;
+
 
     bool m_dirty;
 

src/States/Editor/PythonTokenizer.cpp

@@ -2,4 +2,182 @@
 // Created by Bobby on 9/28/2024.
 //
 
+#include <algorithm>
+#include <iostream>
 #include "PythonTokenizer.h"
+
+PythonTokenizer::PythonTokenizer() {
+    m_keywords = {"False", "None", "True", "and", "as", "assert", "async", "await", "break", "class", "continue", "def",
+                  "del", "elif", "else", "except", "finally", "for", "from", "global", "if", "import", "in", "is", "lambda",
+                  "nonlocal", "not", "or", "pass", "raise", "return", "try", "while", "with", "yield", "self"};
+
+    m_builtins = {"abs", "all", "any", "ascii", "bin", "bool", "breakpoint", "bytearray", "bytes", "callable", "chr", "classmethod",
+                  "compile", "complex", "delattr", "dict", "dir", "divmod", "enumerate", "eval", "exec", "filter", "float", "format",
+                  "frozenset", "getattr", "globals", "hasattr", "hash", "help", "hex", "id", "input", "int", "isinstance", "issubclass",
+                  "iter", "len", "list", "locals", "map", "max", "memoryview", "min", "next", "object", "oct", "open", "ord", "pow",
+                  "print", "property", "range", "repr", "reversed", "round", "set", "setattr", "slice", "sorted", "staticmethod",
+                  "str", "sum", "super", "tuple", "type", "vars", "zip", "__import__", "await", "__init__"};
+}
+
+std::vector<Token> PythonTokenizer::tokenizeLine(const std::string &line) {
+    m_currentPos = 0;
+    std::vector<Token> tokens;
+
+    while(m_currentPos < line.length()){
+        char currentChar = line[m_currentPos];
+
+        if(std::isspace(currentChar))
+        {
+            std::string whitespace = readWhitespace(line);
+            tokens.emplace_back(TokenType::WhiteSpace, whitespace);
+        }
+        else if(std::isalpha(currentChar) || currentChar == '_')
+        {
+
+            std::string identifier = readIdentifier(line);
+            TokenType type;
+            if(isKeyword(identifier)){
+                type = TokenType::Keyword;
+            }else if(isBuiltin(identifier)){
+                type = TokenType::Builtin;
+            }else{
+                type = TokenType::Identifier;
+            }
+            tokens.emplace_back(type, identifier);
+        }
+        else if(currentChar == '"')
+        {
+            std::string identifier = readStringLiteral(line);
+            tokens.emplace_back(TokenType::StringLiteral, identifier);
+        }
+        else if(std::isdigit(currentChar) || currentChar == '.')
+        {
+            std::string numericLiteral = readNumericLiteral(line);
+            if(numericLiteral.empty())
+            { // tested the . for any valid numbers, it failed so punctuation it is.
+                std::string punct(1, currentChar);
+                tokens.emplace_back(TokenType::Punctuation, punct);
+                m_currentPos++;
+            }
+            tokens.emplace_back(TokenType::NumericalLiteral, numericLiteral);
+        }
+        else if(isPunctuation(currentChar))
+        {
+            std::string punct(1, currentChar);
+            tokens.emplace_back(TokenType::Punctuation, punct);
+            m_currentPos++;
+        }
+        else if(isOperator(currentChar))
+        {
+            std::string oper(1, currentChar);
+            tokens.emplace_back(TokenType::Operator, oper);
+            m_currentPos++;
+        }
+        else if(currentChar == '#')
+        {
+            std::string comment = readComment(line);
+            tokens.emplace_back(TokenType::Comment, comment);
+        }
+        else
+        {
+            tokens.emplace_back(TokenType::Unknown, std::string(1, currentChar));
+            m_currentPos++;
+        }
+    }
+
+    return tokens;
+}
+
+std::string PythonTokenizer::readWhitespace(const std::string& line) {
+    size_t start = m_currentPos;
+    while (m_currentPos < line.length() && std::isspace(line[m_currentPos])) {
+        m_currentPos++;
+    }
+    return line.substr(start, m_currentPos - start);
+}
+
+std::string PythonTokenizer::readIdentifier(const std::string &line) {
+    size_t start = m_currentPos;
+    while (m_currentPos < line.length() && (std::isalnum(line[m_currentPos]) || line[m_currentPos] == '_')) {
+        m_currentPos++;
+    }
+    return line.substr(start, m_currentPos - start);
+}
+
+bool PythonTokenizer::isKeyword(const std::string &identifier) {
+    return std::find(m_keywords.begin(), m_keywords.end(), identifier) != m_keywords.end();
+}
+
+bool PythonTokenizer::isBuiltin(const std::string &identifier) {
+    return std::find(m_builtins.begin(), m_builtins.end(), identifier) != m_builtins.end();;
+}
+
+bool PythonTokenizer::isPunctuation(char c) {
+    return c == '(' || c == ')' || c == '{' || c == '}' || c == '[' || c == ']' || c == ',' || c == '.';
+}
+
+bool PythonTokenizer::isOperator(char c) {
+    return c == '+' || c == '-' || c == '*' || c == '/' || c == '=';
+}
+
+std::string PythonTokenizer::readNumericLiteral(const std::string &line) {
+    size_t start = m_currentPos;
+    bool hasDigitsBeforeDecimal = false;
+    bool hasDigitsAfterDecimal = false;
+
+    // Read the integer part (if it exists)
+    while (m_currentPos < line.length() && std::isdigit(line[m_currentPos])) {
+        m_currentPos++;
+        hasDigitsBeforeDecimal = true;
+    }
+
+    // Check for a decimal point
+    if (m_currentPos < line.length() && line[m_currentPos] == '.') {
+        size_t decimalStart = m_currentPos;
+        m_currentPos++; // Move past the decimal point
+
+        // Ensure there's at least one digit after the decimal point
+        if (m_currentPos < line.length() && std::isdigit(line[m_currentPos])) {
+            hasDigitsAfterDecimal = true; // Mark as valid if we see digits after the decimal
+
+            // Continue reading the fractional part
+            while (m_currentPos < line.length() && std::isdigit(line[m_currentPos])) {
+                m_currentPos++;
+            }
+        } else {
+            // If there are no digits before or after the decimal, treat the decimal as punctuation
+            m_currentPos = decimalStart; // Revert to before the decimal point
+        }
+    }
+
+    // Only return a literal if we've encountered at least one digit
+    if (hasDigitsBeforeDecimal || hasDigitsAfterDecimal) {
+        return line.substr(start, m_currentPos - start);
+    } else {
+        // Handle error case where no valid number was found
+        return "";
+    }
+
+}
+
+std::string PythonTokenizer::readStringLiteral(const std::string &line) {
+    size_t start = m_currentPos;
+    m_currentPos++;
+    while (m_currentPos < line.length() && line[m_currentPos] != '"') {
+        m_currentPos++;
+        std::cout << m_currentPos;
+
+    }
+
+    if(line[m_currentPos] == '"') m_currentPos++;
+
+    return line.substr(start, m_currentPos - start);
+}
+
+std::string PythonTokenizer::readComment(const std::string &line) {
+    size_t start = m_currentPos;
+    while (m_currentPos < line.length()) {
+        m_currentPos++;
+    }
+    return line.substr(start, m_currentPos - start);
+}

src/States/Editor/PythonTokenizer.h

@@ -5,14 +5,18 @@
 #pragma once
 
 #include <string>
+#include <vector>
 
 enum TokenType{
     Keyword,
+    Builtin,
     Identifier,
-    Literal,
+    NumericalLiteral,
+    StringLiteral,
     Operator,
     Punctuation,
-    EndOfFile,
+    WhiteSpace,
+    Comment,
     Unknown
 };
 
@@ -25,5 +29,25 @@ struct Token{
 };
 
 class PythonTokenizer {
+public:
+    PythonTokenizer();
+
+    std::vector<Token> tokenizeLine(const std::string& line);
+
+private:
+    size_t m_currentPos;
+
+    std::vector<std::string> m_keywords;
+    std::vector<std::string> m_builtins;
+
+    bool isKeyword(const std::string& identifier);
+    bool isBuiltin(const std::string& identifier);
+    bool isPunctuation(char c);
+    bool isOperator(char c);
+    std::string readWhitespace(const std::string& line);
+    std::string readIdentifier(const std::string& line);
+    std::string readNumericLiteral(const std::string& line);
+    std::string readStringLiteral(const std::string& line);
+    std::string readComment(const std::string& line);
 
 };