Move to public repo

2023-01-27 11:24:58 +00:00 · 2023-01-27 11:24:58 +00:00 · 7acaa9df76
parent d51583204e
commit 7acaa9df76
10 changed files with 571 additions and 0 deletions
--- a/Lexer.cs
+++ b/Lexer.cs
@ -0,0 +1,217 @@
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace ShintenScript
+{
+    public class Lexer
+    {
+        static readonly (string, Token.Type)[] symbolTokens = new (string, Token.Type)[]
+        {
+            ("==", Token.Type.EQ),
+            (">=", Token.Type.GE),
+            ("<=", Token.Type.LE),
+            ("!=", Token.Type.NE),
+            ("+=", Token.Type.PLUSASSIGN),
+            ("-=", Token.Type.MINUSASSIGN),
+            ("*=", Token.Type.ASTERISKASSIGN),
+            ("/=", Token.Type.SLASHASSIGN),
+
+            ("+", Token.Type.PLUS),
+            ("-", Token.Type.MINUS),
+            ("*", Token.Type.ASTERISK),
+            ("/", Token.Type.SLASH),
+            ("=", Token.Type.ASSIGN),
+            ("(", Token.Type.LPAREN),
+            (")", Token.Type.RPAREN),
+            ("{", Token.Type.LBRACE),
+            ("}", Token.Type.RBRACE),
+            (">", Token.Type.GT),
+            ("<", Token.Type.LT),
+        };
+
+        static readonly HashSet<string> keywords = new HashSet<string>
+        {
+            "if", "while", "fn"
+        };
+
+        string code;
+        int index = 0;
+
+        public Lexer(string code_)
+        {
+            if (code_ == null)
+                throw new ArgumentNullException("code");
+
+            code = code_;
+        }
+
+        Token current = new Token { type = Token.Type.NULL };
+
+        Token ParseOne()
+        {
+            if (EOF())
+                return new Token { type = Token.Type.EOF };
+
+            while (char.IsWhiteSpace(code[index]))
+            {
+                index++;
+
+                if (EOF())
+                    return new Token { type = Token.Type.EOF };
+            }
+
+            foreach ((string symbol, Token.Type token) in symbolTokens)
+            {
+                if (code.Substring(index, symbol.Length) == symbol)
+                {
+                    index += symbol.Length;
+                    return new Token { type = token };
+                }
+            }
+
+            if (char.IsLetter(code[index]))
+            {
+                StringBuilder identifierBuilder = new StringBuilder();
+
+                while (char.IsLetterOrDigit(code[index]))
+                {
+                    identifierBuilder.Append(code[index]);
+                    index++;
+                }
+
+                string identifier = identifierBuilder.ToString();
+
+                if (keywords.Contains(identifier))
+                {
+                    Enum.TryParse(identifier, true, out Token.Type type);
+                    return new Token { type = type };
+                }
+
+                return new Token { type = Token.Type.IDENTIFIER, data = identifier };
+            }
+
+            if (char.IsNumber(code[index]))
+            {
+                StringBuilder numberBuilder = new StringBuilder();
+
+                while (char.IsLetterOrDigit(code[index]) || code[index] == '.' || (code[index] == '-' && (code[index - 1] == 'e' || code[index - 1] == 'E')))
+                {
+                    numberBuilder.Append(code[index]);
+                    index++;
+                }
+
+                string numberString = numberBuilder.ToString();
+                float number;
+
+                if (numberString.Contains("."))
+                {
+                    if (!float.TryParse(numberString, NumberStyles.Float, CultureInfo.InvariantCulture, out number))
+                    {
+                        throw new LexingException($"Invalid float literal \"{numberString}\"");
+                    }
+                }
+
+                else
+                {
+                    if (
+                        (numberString.ToLowerInvariant().StartsWith("0x") && long.TryParse(numberString.Substring(2), NumberStyles.HexNumber, CultureInfo.InvariantCulture, out long integer))
+                        || long.TryParse(numberString, NumberStyles.Number, CultureInfo.InvariantCulture, out integer))
+                    {
+                        number = (float)integer;
+                    }
+                    else
+                    {
+                        throw new LexingException($"Invalid integer literal \"{numberString}\"");
+                    }
+                }
+
+                return new Token { type = Token.Type.NUMBER, data = number };
+            }
+
+            throw new LexingException($"Encountered unexpected character '{code[index]}'");
+        }
+
+        bool EOF()
+        {
+            return index >= code.Length;
+        }
+
+        public Token Current()
+        {
+            if (current.Null())
+            {
+                return current = ParseOne();
+            }
+            else
+            {
+                return current;
+            }
+        }
+
+        public Token Pop()
+        {
+            Token token = Current();
+            current = new Token { type = Token.Type.NULL };
+            return token;
+        }
+
+        public bool Accept(params Token.Type[] types)
+        {
+            Token token = Current();
+
+            foreach (Token.Type type in types) {
+                if (token.type == type)
+                {
+                    return true;
+                }
+            }
+
+            return false;
+        }
+
+        public Token Expect(params Token.Type[] types)
+        {
+            Token token = Pop();
+
+            foreach (Token.Type type in types)
+            {
+                if (token.type == type)
+                {
+                    return token;
+                }
+            }
+
+            if (types.Length == 1)
+            {
+                throw new LexingException($"Encountered {token.type} when expecting {types[0]}");
+            }
+            else if (types.Length == 2)
+            {
+                throw new LexingException($"Encountered {token.type} when expecting one of {types[0]} or {types[1]}");
+            }
+            else
+            {
+                string list = string.Join(", ", types.Take(types.Length - 1));
+                throw new LexingException($"Encountered {token.type} when expecting one of {list}, or {types[types.Length - 1]}");
+            }
+        }
+
+        public IEnumerable<Token> TokenStream()
+        {
+            while (!EOF())
+            {
+                yield return Pop();
+            }
+        }
+
+        public void Print()
+        {
+            foreach (Token token in TokenStream())
+                Console.WriteLine(token);
+        }
+    }
+}
--- a/LexingException.cs
+++ b/LexingException.cs
@ -0,0 +1,16 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace ShintenScript
+{
+    public class LexingException : Exception
+    {
+        public LexingException(string message) : base(message)
+        {
+            
+        }
+    }
+}
--- a/Properties/AssemblyInfo.cs
+++ b/Properties/AssemblyInfo.cs
@ -0,0 +1,36 @@
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyTitle("ShintenScript")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("")]
+[assembly: AssemblyProduct("ShintenScript")]
+[assembly: AssemblyCopyright("Copyright ©  2023")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible
+// to COM components.  If you need to access a type in this assembly from
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("f5926e19-83e0-4821-a9ff-5f34dec8fdc9")]
+
+// Version information for an assembly consists of the following four values:
+//
+//      Major Version
+//      Minor Version
+//      Build Number
+//      Revision
+//
+// You can specify all the values or you can default the Build and Revision Numbers
+// by using the '*' as shown below:
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]
--- a/ShintenScript.csproj
+++ b/ShintenScript.csproj
@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProjectGuid>{F5926E19-83E0-4821-A9FF-5F34DEC8FDC9}</ProjectGuid>
+    <OutputType>Library</OutputType>
+    <AppDesignerFolder>Properties</AppDesignerFolder>
+    <RootNamespace>ShintenScript</RootNamespace>
+    <AssemblyName>ShintenScript</AssemblyName>
+    <TargetFrameworkVersion>v4.7.2</TargetFrameworkVersion>
+    <FileAlignment>512</FileAlignment>
+    <Deterministic>true</Deterministic>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>bin\Debug\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>bin\Release\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="System" />
+    <Reference Include="System.Core" />
+    <Reference Include="System.Xml.Linq" />
+    <Reference Include="System.Data.DataSetExtensions" />
+    <Reference Include="Microsoft.CSharp" />
+    <Reference Include="System.Data" />
+    <Reference Include="System.Net.Http" />
+    <Reference Include="System.Xml" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="Lexer.cs" />
+    <Compile Include="LexingException.cs" />
+    <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="Token.cs" />
+  </ItemGroup>
+  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+</Project>
--- a/ShintenScript.sln
+++ b/ShintenScript.sln
@ -0,0 +1,31 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 16
+VisualStudioVersion = 16.0.33130.400
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ShintenScript", "ShintenScript.csproj", "{F5926E19-83E0-4821-A9FF-5F34DEC8FDC9}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ShintenScriptTest", "ShintenScriptTest\ShintenScriptTest.csproj", "{E21E804A-941B-4992-BBA5-9A9F95FE4F00}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{F5926E19-83E0-4821-A9FF-5F34DEC8FDC9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F5926E19-83E0-4821-A9FF-5F34DEC8FDC9}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F5926E19-83E0-4821-A9FF-5F34DEC8FDC9}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F5926E19-83E0-4821-A9FF-5F34DEC8FDC9}.Release|Any CPU.Build.0 = Release|Any CPU
+		{E21E804A-941B-4992-BBA5-9A9F95FE4F00}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{E21E804A-941B-4992-BBA5-9A9F95FE4F00}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{E21E804A-941B-4992-BBA5-9A9F95FE4F00}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{E21E804A-941B-4992-BBA5-9A9F95FE4F00}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {218E18CB-AD4F-41CB-A3B2-E6E079C82769}
+	EndGlobalSection
+EndGlobal
--- a/ShintenScriptTest/LexerTests.cs
+++ b/ShintenScriptTest/LexerTests.cs
@ -0,0 +1,67 @@
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using ShintenScript;
+using System;
+using System.Linq;
+
+namespace ShintenScriptTest
+{
+    [TestClass]
+    public class LexerTests
+    {
+        [TestMethod]
+        public void TestAllTokens()
+        {
+            Lexer lexer = new Lexer(@"
+                hello
+                5
+                0.5
+                0xCafeBabe
+                1.5e6
+                2.4e-6
+
+                ( ) { }
+                + - * /
+                > < == >= <= !=
+                =
+                += -= *= /=
+
+                if while fn
+            ");
+
+            Assert.IsTrue(lexer.TokenStream().SequenceEqual(new Token[] {
+                new Token { type = Token.Type.IDENTIFIER, data = "hello" },
+                new Token { type = Token.Type.NUMBER, data = 5f },
+                new Token { type = Token.Type.NUMBER, data = 0.5f },
+                new Token { type = Token.Type.NUMBER, data = 3405691582f },
+                new Token { type = Token.Type.NUMBER, data = 1.5e6f },
+                new Token { type = Token.Type.NUMBER, data = 2.4e-6f },
+
+                new Token { type = Token.Type.LPAREN },
+                new Token { type = Token.Type.RPAREN },
+                new Token { type = Token.Type.LBRACE },
+                new Token { type = Token.Type.RBRACE },
+                new Token { type = Token.Type.PLUS },
+                new Token { type = Token.Type.MINUS },
+                new Token { type = Token.Type.ASTERISK },
+                new Token { type = Token.Type.SLASH },
+                new Token { type = Token.Type.GT },
+                new Token { type = Token.Type.LT },
+                new Token { type = Token.Type.EQ },
+                new Token { type = Token.Type.GE },
+                new Token { type = Token.Type.LE },
+                new Token { type = Token.Type.NE },
+                new Token { type = Token.Type.ASSIGN },
+                new Token { type = Token.Type.PLUSASSIGN },
+                new Token { type = Token.Type.MINUSASSIGN },
+                new Token { type = Token.Type.ASTERISKASSIGN },
+                new Token { type = Token.Type.SLASHASSIGN },
+
+                new Token { type = Token.Type.IF },
+                new Token { type = Token.Type.WHILE },
+                new Token { type = Token.Type.FN },
+
+                new Token { type = Token.Type.EOF },
+            }));
+        }
+    }
+}
--- a/ShintenScriptTest/Properties/AssemblyInfo.cs
+++ b/ShintenScriptTest/Properties/AssemblyInfo.cs
@ -0,0 +1,20 @@
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+[assembly: AssemblyTitle("ShintenScriptTest")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("")]
+[assembly: AssemblyProduct("ShintenScriptTest")]
+[assembly: AssemblyCopyright("Copyright ©  2023")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+[assembly: ComVisible(false)]
+
+[assembly: Guid("e21e804a-941b-4992-bba5-9a9f95fe4f00")]
+
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]
--- a/ShintenScriptTest/ShintenScriptTest.csproj
+++ b/ShintenScriptTest/ShintenScriptTest.csproj
@ -0,0 +1,74 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="15.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="..\packages\MSTest.TestAdapter.2.1.2\build\net45\MSTest.TestAdapter.props" Condition="Exists('..\packages\MSTest.TestAdapter.2.1.2\build\net45\MSTest.TestAdapter.props')" />
+  <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProjectGuid>{E21E804A-941B-4992-BBA5-9A9F95FE4F00}</ProjectGuid>
+    <OutputType>Library</OutputType>
+    <AppDesignerFolder>Properties</AppDesignerFolder>
+    <RootNamespace>ShintenScriptTest</RootNamespace>
+    <AssemblyName>ShintenScriptTest</AssemblyName>
+    <TargetFrameworkVersion>v4.7.2</TargetFrameworkVersion>
+    <FileAlignment>512</FileAlignment>
+    <ProjectTypeGuids>{3AC096D0-A1C2-E12C-1390-A8335801FDAB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
+    <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">15.0</VisualStudioVersion>
+    <VSToolsPath Condition="'$(VSToolsPath)' == ''">$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)</VSToolsPath>
+    <ReferencePath>$(ProgramFiles)\Common Files\microsoft shared\VSTT\$(VisualStudioVersion)\UITestExtensionPackages</ReferencePath>
+    <IsCodedUITest>False</IsCodedUITest>
+    <TestProjectType>UnitTest</TestProjectType>
+    <NuGetPackageImportStamp>
+    </NuGetPackageImportStamp>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>bin\Debug\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>bin\Release\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="Microsoft.VisualStudio.TestPlatform.TestFramework, Version=14.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
+      <HintPath>..\packages\MSTest.TestFramework.2.1.2\lib\net45\Microsoft.VisualStudio.TestPlatform.TestFramework.dll</HintPath>
+    </Reference>
+    <Reference Include="Microsoft.VisualStudio.TestPlatform.TestFramework.Extensions, Version=14.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
+      <HintPath>..\packages\MSTest.TestFramework.2.1.2\lib\net45\Microsoft.VisualStudio.TestPlatform.TestFramework.Extensions.dll</HintPath>
+    </Reference>
+    <Reference Include="System" />
+    <Reference Include="System.Core" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="LexerTests.cs" />
+    <Compile Include="Properties\AssemblyInfo.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <None Include="packages.config" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\ShintenScript.csproj">
+      <Project>{F5926E19-83E0-4821-A9FF-5F34DEC8FDC9}</Project>
+      <Name>ShintenScript</Name>
+    </ProjectReference>
+  </ItemGroup>
+  <Import Project="$(VSToolsPath)\TeamTest\Microsoft.TestTools.targets" Condition="Exists('$(VSToolsPath)\TeamTest\Microsoft.TestTools.targets')" />
+  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
+    <PropertyGroup>
+      <ErrorText>This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
+    </PropertyGroup>
+    <Error Condition="!Exists('..\packages\MSTest.TestAdapter.2.1.2\build\net45\MSTest.TestAdapter.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\MSTest.TestAdapter.2.1.2\build\net45\MSTest.TestAdapter.props'))" />
+    <Error Condition="!Exists('..\packages\MSTest.TestAdapter.2.1.2\build\net45\MSTest.TestAdapter.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\MSTest.TestAdapter.2.1.2\build\net45\MSTest.TestAdapter.targets'))" />
+  </Target>
+  <Import Project="..\packages\MSTest.TestAdapter.2.1.2\build\net45\MSTest.TestAdapter.targets" Condition="Exists('..\packages\MSTest.TestAdapter.2.1.2\build\net45\MSTest.TestAdapter.targets')" />
+</Project>
--- a/ShintenScriptTest/packages.config
+++ b/ShintenScriptTest/packages.config
@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="utf-8"?>
+<packages>
+  <package id="MSTest.TestAdapter" version="2.1.2" targetFramework="net472" />
+  <package id="MSTest.TestFramework" version="2.1.2" targetFramework="net472" />
+</packages>
--- a/Token.cs
+++ b/Token.cs
@ -0,0 +1,55 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace ShintenScript
+{
+    public struct Token
+    {
+        public enum Type
+        {
+            NULL,
+            EOF,
+            IDENTIFIER,
+            NUMBER,
+
+            LPAREN, RPAREN, LBRACE, RBRACE,
+            PLUS, MINUS, ASTERISK, SLASH,
+            GT, LT, EQ, GE, LE, NE,
+            ASSIGN,
+            PLUSASSIGN, MINUSASSIGN, ASTERISKASSIGN, SLASHASSIGN,
+
+            IF, WHILE, FN
+        }
+
+        public Type type;
+        public object data;
+
+        public bool Null()
+        {
+            return type == Type.NULL;
+        }
+
+        public override int GetHashCode()
+        {
+            return (int)type;
+        }
+
+        public override bool Equals(object obj)
+        {
+            if (obj is Token other)
+            {
+                return type == other.type && ((data == null && other.data == null) || (data != null && other.data != null && data.Equals(other.data)));
+            }
+
+            return false;
+        }
+
+        public override string ToString()
+        {
+            return $"{type} {data?.ToString() ?? "null"}";
+        }
+    }
+}