using System.Collections; using System.Text; #nullable disable namespace InteractiveReadLine.Tokenizing { /// /// The TokenizedLine class represents a line of text processed by a lexer and split into a sequence of /// tokens. /// /// /// These tokens in order contain every character of text in the original line, such that combining /// them in sequence would recreate the original string. Additionally, the line's original cursor position /// is preserved and contained by one of the tokens. The tokens can be walked in its entirety in either /// direction by navigation properties on the tokens themselves, or iterated through in the line. /// public class TokenizedLine : IReadOnlyList { private readonly List _tokens; private int _cursor; public TokenizedLine() { _tokens = []; } /// /// Gets the first token in the sequence, or null if the sequence is empty /// public IToken First => _tokens.FirstOrDefault(); /// /// Gets the last token in the sequence, or null if the sequence is empty /// public IToken Last => _tokens.LastOrDefault(); /// /// Gets the first non-hidden token in the sequence, or null if none exist /// public IToken FirstNonHidden => _tokens.FirstOrDefault()?.FirstNonHidden; /// /// Gets the combined text of all of the tokens in the sequence, which will match the original line of text /// before it was split apart by the lexer /// public string Text => Token.BuildText(_tokens); /// /// Gets an enumerator which can be used to iterate through the tokens /// /// public IEnumerator GetEnumerator() => _tokens.GetEnumerator(); /// /// Gets an enumerator which can be used to iterate through the tokens /// /// IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); /// /// Gets the number of tokens in the sequence /// public int Count => _tokens.Count; /// /// Gets the token at the specified index in the sequence /// /// Index in the token sequence /// public IToken this[int index] => _tokens[index]; /// /// Gets the token which currently contains the cursor /// public IToken CursorToken => _tokens.FirstOrDefault(x => x.Cursor != null); /// /// Gets the index number of the token which currently contains the cursor /// public int CursorTokenIndex => _tokens.IndexOf(_tokens.FirstOrDefault(x => x.Cursor != null)); /// /// Gets the overall index of the cursor in the combined text /// public int Cursor { get => _cursor; set { _cursor = value; int textLen = this.Text.Length; if (value > textLen) _cursor = textLen; } } /// /// Add a token to the sequence, making all of the necessary links /// /// The text contained by the token /// Whether or not this is a hidden token /// Null if the token does not contain the cursor, otherwise the index of the cursor as an offset from the first character in the token text /// An optional type code which can be accessed by a consumer of the token public void Add(string text, bool isHidden, int? cursor=null, int typeCode=0) { var newToken = new Token(text, isHidden, this, typeCode); if (_tokens.Count != 0) { _tokens.Last().Next = newToken; newToken.Previous = _tokens.Last(); } _tokens.Add(newToken); if (cursor != null) newToken.Cursor = cursor; } /// /// Internal implementation of the token /// /// /// /// Tokens and the TokenizedLine require a lot of internal plumbing to work correctly, and don't necessarily /// have a meaning or purpose except in conjunction with each other. To make sure that the TokenizedLine.Add /// method is the obvious way of producing tokens and connecting them, and that tokens are not created /// independent from a TokenizedLine, this class is hidden from consumers and exposed only through the /// IToken interface. /// private class Token(string text, bool isHidden, TokenizedLine parent, int typeCode) : IToken { public int TypeCode { get; } = typeCode; public string Text { get => text; set { int? cursorMove = null; // Adjust the cursor if the cursor lies in this token if (Cursor != null && Cursor > value.Length) { Cursor = value.Length; } else { // Check if the cursor is after this token, and if so adjust it accordingly var beforeLen = TextBefore().Length; if (beforeLen + text.Length - 1 < parent.Cursor) { var delta = value.Length - text.Length; cursorMove = parent.Cursor + delta; } } text = value; if (cursorMove != null) parent.Cursor = (int) cursorMove; } } public int? Cursor { get { int lengthBefore = this.TextBefore().Length; if (parent.Cursor < lengthBefore) return null; int offset = parent.Cursor - lengthBefore; if (offset == Text.Length && Next?.IsHidden == false) { return null; } if (offset <= Text.Length) return offset; return null; } set { if (value == null || value < 0 || value > Text.Length) return; parent.Cursor = this.TextBefore().Length + (int) value; } } public Token Next { get; set; } public Token Previous { get; set; } public IToken PreviousNotHidden => this.Previous?.ThisOrPrevIfHidden(); public IToken NextNotHidden => this.Next?.ThisOrNextIfHidden(); IToken IToken.Next => this.Next; IToken IToken.Previous => this.Previous; public Token First => this.Previous == null ? this : this.Previous.First; public Token FirstNonHidden => this.First.ThisOrNextIfHidden(); public bool IsHidden { get; set; } = isHidden; public int? DistanceTo(IToken other, bool ignoreHidden = false) { if (other is not Token token) return null; var gap = Next?.ForwardTo(token, []); if (ignoreHidden) return gap?.Where(x => !x.IsHidden).Count(); else return gap?.Count; } private List ForwardTo(Token other, List gap) { gap.Add(this); if (other == this) return gap; else { return this.Next?.ForwardTo(other, gap); } } private Token[] TokensBefore() { var tokens = new List(); var first = this.First; var pointer = first; while (pointer != this) { tokens.Add(pointer); pointer = pointer.Next; } return [.. tokens]; } private string TextBefore() { return BuildText(this.TokensBefore()); } private Token ThisOrNextIfHidden() { if (this.IsHidden) return this.Next?.ThisOrNextIfHidden(); else return this; } private Token ThisOrPrevIfHidden() { if (this.IsHidden) return this.Previous?.ThisOrPrevIfHidden(); else return this; } public static string BuildText(IEnumerable tokens) { var builder = new StringBuilder(); foreach (var token in tokens) { builder.Append(token.Text); } return builder.ToString(); } } } }