diff --git a/python/main.py b/python/main.py index abcc9f4..ee5a7d7 100644 --- a/python/main.py +++ b/python/main.py @@ -2,6 +2,7 @@ from scene import Scene from particles import Particles from triangles import Triangles +# Palette class (inherits from scene) class Palette(Scene): def __init__(self): diff --git a/python/syntaxTest.py b/python/syntaxTest.py new file mode 100644 index 0000000..a151317 --- /dev/null +++ b/python/syntaxTest.py @@ -0,0 +1,15 @@ + +#asdlkjasd +test #this is a test + +print("test") + + + 9 . 42 .12 9.1.1 80. 80.0123 + + "" asd + " + "Test" + + + diff --git a/src/Pycron.h b/src/Pycron.h index c4b4d3a..28ad018 100644 --- a/src/Pycron.h +++ b/src/Pycron.h @@ -31,6 +31,8 @@ public: void StartGameLoop(); void bindMethods(); + static std::string loadFileToString(const std::string& filename); + static pkpy::PyObject* getRandomNumber(pkpy::VM* vm, pkpy::ArgsView args); static pkpy::PyObject* getSin(pkpy::VM* vm, pkpy::ArgsView args); static pkpy::PyObject* getCos(pkpy::VM* vm, pkpy::ArgsView args); @@ -39,7 +41,5 @@ public: static pkpy::PyObject* getKeyDown(pkpy::VM* vm, pkpy::ArgsView args); static pkpy::PyObject* getMousePressed(pkpy::VM* vm, pkpy::ArgsView args); static pkpy::PyObject* getMouseDown(pkpy::VM* vm, pkpy::ArgsView args); - - static std::string loadFileToString(const std::string& filename); }; diff --git a/src/States/Editor/EditorState.cpp b/src/States/Editor/EditorState.cpp index 1a01e4e..00c2c61 100644 --- a/src/States/Editor/EditorState.cpp +++ b/src/States/Editor/EditorState.cpp @@ -17,14 +17,24 @@ EditorState::EditorState(pkpy::VM *vm, Graphics *graphics) : m_vm(vm), m_graphic std::string randomSource = Pycron::loadFileToString("../python/main.py"); - m_baseBackgroundColor = 59; - m_baseTextColor = 51; - m_lineNumberBackgroundColor = 58; - m_lineNumberTextColor = 61; + m_baseBackgroundColor = 56; + m_baseTextColor = 63; + m_lineNumberBackgroundColor = 57; + m_lineNumberTextColor = 6; + m_unknownTextColor = 4; + m_identifierTextColor = 43; + m_keywordTextColor = 31; + m_builtinTextColor = 23; + m_numericalLiteralTextColor = 32; + m_stringLiteralTextColor = 6; + m_punctuationTextColor = 52; + m_operatorTextColor = 45; + m_commentTextColor = 60; - m_topLetterSpacing = 1; + + m_topLetterSpacing = 0; m_bottomLetterSpacing = 1; - m_leftLetterSpacing = 1; + m_leftLetterSpacing = 0; m_rightLetterSpacing = 1; m_charWidth = m_leftLetterSpacing + m_graphics->GetCurrentFontWidth() + m_rightLetterSpacing; // Final size of char with spacing. If the literal width of the font is n, the final width is n + spacing. @@ -37,13 +47,14 @@ EditorState::EditorState(pkpy::VM *vm, Graphics *graphics) : m_vm(vm), m_graphic m_foregroundIndexBuffer = std::vector(m_textWidth * m_textHeight); m_backgroundIndexBuffer = std::vector(m_textWidth * m_textHeight); + m_dirtyFlags = std::vector(m_textHeight); + Clear(); - - std::string testText = "This is a test\n[Test]\nThis is the end of the test."; - LoadStringToBuffer(randomSource); + auto tokens = m_pythonTokenizer->tokenizeLine("for i in range(100):"); + } EditorState::~EditorState() { @@ -52,13 +63,14 @@ EditorState::~EditorState() { void EditorState::Draw() { m_graphics->Clear(0); - + + int bg = 28; for (int i = 0; i < m_characterBuffer.size(); ++i) { int x = (i % m_textWidth) * m_charWidth; int y = (i / m_textWidth) * m_charHeight; m_graphics->Rect(x, y, m_charWidth, m_charHeight, m_backgroundIndexBuffer[i]); - m_graphics->Char(m_characterBuffer[i], x + m_leftLetterSpacing + 1, y + m_topLetterSpacing + 1, 0); + m_graphics->Char(m_characterBuffer[i], x + m_leftLetterSpacing + 1, y + m_topLetterSpacing + 1, bg); m_graphics->Char(m_characterBuffer[i], x + m_leftLetterSpacing, y + m_topLetterSpacing, m_foregroundIndexBuffer[i]); } @@ -66,12 +78,42 @@ void EditorState::Draw() { Clear(); m_dirty = false; for (int i = 0; i < m_text.size(); ++i) { + // Line numbers TODO: maybe not have this as part of the buffer, instead as a custom bar. (Allows for more custom functionality such as bookmarks) std::string lineNumber = std::to_string(i); int size = 2; int diff = size - (int)lineNumber.size(); if(diff > 0) lineNumber = std::string(diff, ' ') + lineNumber; Text(lineNumber, 0, i, m_lineNumberTextColor, m_lineNumberBackgroundColor); - Text(m_text[i], size, i, m_baseTextColor, m_baseBackgroundColor); + + // Text handling + auto tokens = m_pythonTokenizer->tokenizeLine(m_text[i]); + + int currentPos = 0; + for (int j = 0; j < tokens.size(); ++j) { + int color = m_baseTextColor; + TokenType type = tokens[j].type; + if(type == TokenType::Identifier){ + color = m_identifierTextColor; + }else if(type == TokenType::Keyword){ + color = m_keywordTextColor; + }else if(type == TokenType::Builtin){ + color = m_builtinTextColor; + }else if(type == TokenType::NumericalLiteral){ + color = m_numericalLiteralTextColor; + }else if(type == TokenType::StringLiteral){ + color = m_stringLiteralTextColor; + }else if(type == TokenType::Punctuation){ + color = m_punctuationTextColor; + }else if(type == TokenType::Operator){ + color = m_operatorTextColor; + }else if(type == TokenType::Comment){ + color = m_commentTextColor; + }else if(type == TokenType::Unknown){ + color = m_baseTextColor; + } + Text(tokens[j].value, size + currentPos, i, color, m_baseBackgroundColor); + currentPos += tokens[j].value.size(); + } } } diff --git a/src/States/Editor/EditorState.h b/src/States/Editor/EditorState.h index 2d2d0d1..0d30933 100644 --- a/src/States/Editor/EditorState.h +++ b/src/States/Editor/EditorState.h @@ -34,12 +34,23 @@ private: // Text Data std::vector m_text; + std::vector m_dirtyFlags; // Theming uint8_t m_baseBackgroundColor; uint8_t m_baseTextColor; uint8_t m_lineNumberBackgroundColor; uint8_t m_lineNumberTextColor; + uint8_t m_unknownTextColor; + uint8_t m_identifierTextColor; + uint8_t m_keywordTextColor; + uint8_t m_builtinTextColor; + uint8_t m_numericalLiteralTextColor; + uint8_t m_stringLiteralTextColor; + uint8_t m_punctuationTextColor; + uint8_t m_operatorTextColor; + uint8_t m_commentTextColor; + bool m_dirty; diff --git a/src/States/Editor/PythonTokenizer.cpp b/src/States/Editor/PythonTokenizer.cpp index e5159bb..ea4b9fb 100644 --- a/src/States/Editor/PythonTokenizer.cpp +++ b/src/States/Editor/PythonTokenizer.cpp @@ -2,4 +2,182 @@ // Created by Bobby on 9/28/2024. // +#include +#include #include "PythonTokenizer.h" + +PythonTokenizer::PythonTokenizer() { + m_keywords = {"False", "None", "True", "and", "as", "assert", "async", "await", "break", "class", "continue", "def", + "del", "elif", "else", "except", "finally", "for", "from", "global", "if", "import", "in", "is", "lambda", + "nonlocal", "not", "or", "pass", "raise", "return", "try", "while", "with", "yield", "self"}; + + m_builtins = {"abs", "all", "any", "ascii", "bin", "bool", "breakpoint", "bytearray", "bytes", "callable", "chr", "classmethod", + "compile", "complex", "delattr", "dict", "dir", "divmod", "enumerate", "eval", "exec", "filter", "float", "format", + "frozenset", "getattr", "globals", "hasattr", "hash", "help", "hex", "id", "input", "int", "isinstance", "issubclass", + "iter", "len", "list", "locals", "map", "max", "memoryview", "min", "next", "object", "oct", "open", "ord", "pow", + "print", "property", "range", "repr", "reversed", "round", "set", "setattr", "slice", "sorted", "staticmethod", + "str", "sum", "super", "tuple", "type", "vars", "zip", "__import__", "await", "__init__"}; +} + +std::vector PythonTokenizer::tokenizeLine(const std::string &line) { + m_currentPos = 0; + std::vector tokens; + + while(m_currentPos < line.length()){ + char currentChar = line[m_currentPos]; + + if(std::isspace(currentChar)) + { + std::string whitespace = readWhitespace(line); + tokens.emplace_back(TokenType::WhiteSpace, whitespace); + } + else if(std::isalpha(currentChar) || currentChar == '_') + { + + std::string identifier = readIdentifier(line); + TokenType type; + if(isKeyword(identifier)){ + type = TokenType::Keyword; + }else if(isBuiltin(identifier)){ + type = TokenType::Builtin; + }else{ + type = TokenType::Identifier; + } + tokens.emplace_back(type, identifier); + } + else if(currentChar == '"') + { + std::string identifier = readStringLiteral(line); + tokens.emplace_back(TokenType::StringLiteral, identifier); + } + else if(std::isdigit(currentChar) || currentChar == '.') + { + std::string numericLiteral = readNumericLiteral(line); + if(numericLiteral.empty()) + { // tested the . for any valid numbers, it failed so punctuation it is. + std::string punct(1, currentChar); + tokens.emplace_back(TokenType::Punctuation, punct); + m_currentPos++; + } + tokens.emplace_back(TokenType::NumericalLiteral, numericLiteral); + } + else if(isPunctuation(currentChar)) + { + std::string punct(1, currentChar); + tokens.emplace_back(TokenType::Punctuation, punct); + m_currentPos++; + } + else if(isOperator(currentChar)) + { + std::string oper(1, currentChar); + tokens.emplace_back(TokenType::Operator, oper); + m_currentPos++; + } + else if(currentChar == '#') + { + std::string comment = readComment(line); + tokens.emplace_back(TokenType::Comment, comment); + } + else + { + tokens.emplace_back(TokenType::Unknown, std::string(1, currentChar)); + m_currentPos++; + } + } + + return tokens; +} + +std::string PythonTokenizer::readWhitespace(const std::string& line) { + size_t start = m_currentPos; + while (m_currentPos < line.length() && std::isspace(line[m_currentPos])) { + m_currentPos++; + } + return line.substr(start, m_currentPos - start); +} + +std::string PythonTokenizer::readIdentifier(const std::string &line) { + size_t start = m_currentPos; + while (m_currentPos < line.length() && (std::isalnum(line[m_currentPos]) || line[m_currentPos] == '_')) { + m_currentPos++; + } + return line.substr(start, m_currentPos - start); +} + +bool PythonTokenizer::isKeyword(const std::string &identifier) { + return std::find(m_keywords.begin(), m_keywords.end(), identifier) != m_keywords.end(); +} + +bool PythonTokenizer::isBuiltin(const std::string &identifier) { + return std::find(m_builtins.begin(), m_builtins.end(), identifier) != m_builtins.end();; +} + +bool PythonTokenizer::isPunctuation(char c) { + return c == '(' || c == ')' || c == '{' || c == '}' || c == '[' || c == ']' || c == ',' || c == '.'; +} + +bool PythonTokenizer::isOperator(char c) { + return c == '+' || c == '-' || c == '*' || c == '/' || c == '='; +} + +std::string PythonTokenizer::readNumericLiteral(const std::string &line) { + size_t start = m_currentPos; + bool hasDigitsBeforeDecimal = false; + bool hasDigitsAfterDecimal = false; + + // Read the integer part (if it exists) + while (m_currentPos < line.length() && std::isdigit(line[m_currentPos])) { + m_currentPos++; + hasDigitsBeforeDecimal = true; + } + + // Check for a decimal point + if (m_currentPos < line.length() && line[m_currentPos] == '.') { + size_t decimalStart = m_currentPos; + m_currentPos++; // Move past the decimal point + + // Ensure there's at least one digit after the decimal point + if (m_currentPos < line.length() && std::isdigit(line[m_currentPos])) { + hasDigitsAfterDecimal = true; // Mark as valid if we see digits after the decimal + + // Continue reading the fractional part + while (m_currentPos < line.length() && std::isdigit(line[m_currentPos])) { + m_currentPos++; + } + } else { + // If there are no digits before or after the decimal, treat the decimal as punctuation + m_currentPos = decimalStart; // Revert to before the decimal point + } + } + + // Only return a literal if we've encountered at least one digit + if (hasDigitsBeforeDecimal || hasDigitsAfterDecimal) { + return line.substr(start, m_currentPos - start); + } else { + // Handle error case where no valid number was found + return ""; + } + +} + +std::string PythonTokenizer::readStringLiteral(const std::string &line) { + size_t start = m_currentPos; + m_currentPos++; + while (m_currentPos < line.length() && line[m_currentPos] != '"') { + m_currentPos++; + std::cout << m_currentPos; + + } + + if(line[m_currentPos] == '"') m_currentPos++; + + return line.substr(start, m_currentPos - start); +} + +std::string PythonTokenizer::readComment(const std::string &line) { + size_t start = m_currentPos; + while (m_currentPos < line.length()) { + m_currentPos++; + } + return line.substr(start, m_currentPos - start); +} diff --git a/src/States/Editor/PythonTokenizer.h b/src/States/Editor/PythonTokenizer.h index d671e28..daef8d8 100644 --- a/src/States/Editor/PythonTokenizer.h +++ b/src/States/Editor/PythonTokenizer.h @@ -5,14 +5,18 @@ #pragma once #include +#include enum TokenType{ Keyword, + Builtin, Identifier, - Literal, + NumericalLiteral, + StringLiteral, Operator, Punctuation, - EndOfFile, + WhiteSpace, + Comment, Unknown }; @@ -25,5 +29,25 @@ struct Token{ }; class PythonTokenizer { +public: + PythonTokenizer(); + + std::vector tokenizeLine(const std::string& line); + +private: + size_t m_currentPos; + + std::vector m_keywords; + std::vector m_builtins; + + bool isKeyword(const std::string& identifier); + bool isBuiltin(const std::string& identifier); + bool isPunctuation(char c); + bool isOperator(char c); + std::string readWhitespace(const std::string& line); + std::string readIdentifier(const std::string& line); + std::string readNumericLiteral(const std::string& line); + std::string readStringLiteral(const std::string& line); + std::string readComment(const std::string& line); };