using System.Collections;
using System.Text;
#nullable disable
namespace InteractiveReadLine.Tokenizing
{
///
/// The TokenizedLine class represents a line of text processed by a lexer and split into a sequence of
/// tokens.
///
///
/// These tokens in order contain every character of text in the original line, such that combining
/// them in sequence would recreate the original string. Additionally, the line's original cursor position
/// is preserved and contained by one of the tokens. The tokens can be walked in its entirety in either
/// direction by navigation properties on the tokens themselves, or iterated through in the line.
///
public class TokenizedLine : IReadOnlyList
{
private readonly List _tokens;
private int _cursor;
public TokenizedLine()
{
_tokens = [];
}
///
/// Gets the first token in the sequence, or null if the sequence is empty
///
public IToken First => _tokens.FirstOrDefault();
///
/// Gets the last token in the sequence, or null if the sequence is empty
///
public IToken Last => _tokens.LastOrDefault();
///
/// Gets the first non-hidden token in the sequence, or null if none exist
///
public IToken FirstNonHidden => _tokens.FirstOrDefault()?.FirstNonHidden;
///
/// Gets the combined text of all of the tokens in the sequence, which will match the original line of text
/// before it was split apart by the lexer
///
public string Text => Token.BuildText(_tokens);
///
/// Gets an enumerator which can be used to iterate through the tokens
///
///
public IEnumerator GetEnumerator() => _tokens.GetEnumerator();
///
/// Gets an enumerator which can be used to iterate through the tokens
///
///
IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();
///
/// Gets the number of tokens in the sequence
///
public int Count => _tokens.Count;
///
/// Gets the token at the specified index in the sequence
///
/// Index in the token sequence
///
public IToken this[int index] => _tokens[index];
///
/// Gets the token which currently contains the cursor
///
public IToken CursorToken => _tokens.FirstOrDefault(x => x.Cursor != null);
///
/// Gets the index number of the token which currently contains the cursor
///
public int CursorTokenIndex => _tokens.IndexOf(_tokens.FirstOrDefault(x => x.Cursor != null));
///
/// Gets the overall index of the cursor in the combined text
///
public int Cursor
{
get => _cursor;
set
{
_cursor = value;
int textLen = this.Text.Length;
if (value > textLen)
_cursor = textLen;
}
}
///
/// Add a token to the sequence, making all of the necessary links
///
/// The text contained by the token
/// Whether or not this is a hidden token
/// Null if the token does not contain the cursor, otherwise the index of the cursor as an offset from the first character in the token text
/// An optional type code which can be accessed by a consumer of the token
public void Add(string text, bool isHidden, int? cursor=null, int typeCode=0)
{
var newToken = new Token(text, isHidden, this, typeCode);
if (_tokens.Count != 0)
{
_tokens.Last().Next = newToken;
newToken.Previous = _tokens.Last();
}
_tokens.Add(newToken);
if (cursor != null)
newToken.Cursor = cursor;
}
///
/// Internal implementation of the token
///
///
///
/// Tokens and the TokenizedLine require a lot of internal plumbing to work correctly, and don't necessarily
/// have a meaning or purpose except in conjunction with each other. To make sure that the TokenizedLine.Add
/// method is the obvious way of producing tokens and connecting them, and that tokens are not created
/// independent from a TokenizedLine, this class is hidden from consumers and exposed only through the
/// IToken interface.
///
private class Token(string text, bool isHidden, TokenizedLine parent, int typeCode) : IToken
{
public int TypeCode { get; } = typeCode;
public string Text
{
get => text;
set
{
int? cursorMove = null;
// Adjust the cursor if the cursor lies in this token
if (Cursor != null && Cursor > value.Length)
{
Cursor = value.Length;
}
else
{
// Check if the cursor is after this token, and if so adjust it accordingly
var beforeLen = TextBefore().Length;
if (beforeLen + text.Length - 1 < parent.Cursor)
{
var delta = value.Length - text.Length;
cursorMove = parent.Cursor + delta;
}
}
text = value;
if (cursorMove != null)
parent.Cursor = (int) cursorMove;
}
}
public int? Cursor
{
get
{
int lengthBefore = this.TextBefore().Length;
if (parent.Cursor < lengthBefore)
return null;
int offset = parent.Cursor - lengthBefore;
if (offset == Text.Length && Next?.IsHidden == false)
{
return null;
}
if (offset <= Text.Length)
return offset;
return null;
}
set
{
if (value == null || value < 0 || value > Text.Length)
return;
parent.Cursor = this.TextBefore().Length + (int) value;
}
}
public Token Next { get; set; }
public Token Previous { get; set; }
public IToken PreviousNotHidden => this.Previous?.ThisOrPrevIfHidden();
public IToken NextNotHidden => this.Next?.ThisOrNextIfHidden();
IToken IToken.Next => this.Next;
IToken IToken.Previous => this.Previous;
public Token First => this.Previous == null ? this : this.Previous.First;
public Token FirstNonHidden => this.First.ThisOrNextIfHidden();
public bool IsHidden { get; set; } = isHidden;
public int? DistanceTo(IToken other, bool ignoreHidden = false)
{
if (other is not Token token)
return null;
var gap = Next?.ForwardTo(token, []);
if (ignoreHidden)
return gap?.Where(x => !x.IsHidden).Count();
else
return gap?.Count;
}
private List ForwardTo(Token other, List gap)
{
gap.Add(this);
if (other == this)
return gap;
else
{
return this.Next?.ForwardTo(other, gap);
}
}
private Token[] TokensBefore()
{
var tokens = new List();
var first = this.First;
var pointer = first;
while (pointer != this)
{
tokens.Add(pointer);
pointer = pointer.Next;
}
return [.. tokens];
}
private string TextBefore()
{
return BuildText(this.TokensBefore());
}
private Token ThisOrNextIfHidden()
{
if (this.IsHidden)
return this.Next?.ThisOrNextIfHidden();
else
return this;
}
private Token ThisOrPrevIfHidden()
{
if (this.IsHidden)
return this.Previous?.ThisOrPrevIfHidden();
else
return this;
}
public static string BuildText(IEnumerable tokens)
{
var builder = new StringBuilder();
foreach (var token in tokens)
{
builder.Append(token.Text);
}
return builder.ToString();
}
}
}
}