1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283using System.Collections;

using System.Text;

#nullable disable

namespace InteractiveReadLine.Tokenizing
{
    /// <summary>
    /// The TokenizedLine class represents a line of text processed by a lexer and split into a sequence of
    /// tokens.
    /// </summary>
    /// <remarks>
    /// These tokens in order contain every character of text in the original line, such that combining
    /// them in sequence would recreate the original string. Additionally, the line's original cursor position
    /// is preserved and contained by one of the tokens. The tokens can be walked in its entirety in either
    /// direction by navigation properties on the tokens themselves, or iterated through in the line.
    /// </remarks>
    public class TokenizedLine : IReadOnlyList<IToken>
    {
        private readonly List<Token> _tokens;
        private int _cursor;

        public TokenizedLine()
        {
            _tokens = [];
        }

        /// <summary>
        /// Gets the first token in the sequence, or null if the sequence is empty
        /// </summary>
        public IToken First => _tokens.FirstOrDefault();

        /// <summary>
        /// Gets the last token in the sequence, or null if the sequence is empty
        /// </summary>
        public IToken Last => _tokens.LastOrDefault();

        /// <summary>
        /// Gets the first non-hidden token in the sequence, or null if none exist
        /// </summary>
        public IToken FirstNonHidden => _tokens.FirstOrDefault()?.FirstNonHidden;

        /// <summary>
        /// Gets the combined text of all of the tokens in the sequence, which will match the original line of text
        /// before it was split apart by the lexer
        /// </summary>
        public string Text => Token.BuildText(_tokens);

        /// <summary>
        /// Gets an enumerator which can be used to iterate through the tokens
        /// </summary>
        /// <returns></returns>
        public IEnumerator<IToken> GetEnumerator() => _tokens.GetEnumerator();

        /// <summary>
        /// Gets an enumerator which can be used to iterate through the tokens
        /// </summary>
        /// <returns></returns>
        IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();

        /// <summary>
        /// Gets the number of tokens in the sequence
        /// </summary>
        public int Count => _tokens.Count;

        /// <summary>
        /// Gets the token at the specified index in the sequence
        /// </summary>
        /// <param name="index">Index in the token sequence</param>
        /// <returns></returns>
        public IToken this[int index] => _tokens[index];

        /// <summary>
        /// Gets the token which currently contains the cursor
        /// </summary>
        public IToken CursorToken => _tokens.FirstOrDefault(x => x.Cursor != null);

        /// <summary>
        /// Gets the index number of the token which currently contains the cursor
        /// </summary>
        public int CursorTokenIndex => _tokens.IndexOf(_tokens.FirstOrDefault(x => x.Cursor != null));

        /// <summary>
        /// Gets the overall index of the cursor in the combined text
        /// </summary>
        public int Cursor
        {
            get => _cursor;
            set
            {
                _cursor = value;
                
                int textLen = this.Text.Length;
                if (value > textLen)
                    _cursor = textLen;
            }
        }

        /// <summary>
        /// Add a token to the sequence, making all of the necessary links
        /// </summary>
        /// <param name="text">The text contained by the token</param>
        /// <param name="isHidden">Whether or not this is a hidden token</param>
        /// <param name="cursor">Null if the token does not contain the cursor, otherwise the index of the cursor as an offset from the first character in the token text</param>
        /// <param name="typeCode">An optional type code which can be accessed by a consumer of the token</param>
        public void Add(string text, bool isHidden, int? cursor=null, int typeCode=0)
        {
            var newToken = new Token(text, isHidden, this, typeCode);

            if (_tokens.Count != 0)
            {
                _tokens.Last().Next = newToken;
                newToken.Previous = _tokens.Last();
            }

            _tokens.Add(newToken);

            if (cursor != null)
                newToken.Cursor = cursor;
        }

        /// <summary>
        /// Internal implementation of the token
        /// </summary>
        /// 
        /// <remarks>
        /// Tokens and the TokenizedLine require a lot of internal plumbing to work correctly, and don't necessarily
        /// have a meaning or purpose except in conjunction with each other.  To make sure that the TokenizedLine.Add
        /// method is the obvious way of producing tokens and connecting them, and that tokens are not created
        /// independent from a TokenizedLine, this class is hidden from consumers and exposed only through the
        /// IToken interface.  
        /// </remarks>
        private class Token(string text, bool isHidden, TokenizedLine parent, int typeCode) : IToken
        {
			public int TypeCode { get; } = typeCode;

			public string Text
            {
                get => text;
                set
                {
                    int? cursorMove = null;

                    // Adjust the cursor if the cursor lies in this token
                    if (Cursor != null && Cursor > value.Length)
                    {
                        Cursor = value.Length;
                    }
                    else
                    {
                        // Check if the cursor is after this token, and if so adjust it accordingly
                        var beforeLen = TextBefore().Length;
                        if (beforeLen + text.Length - 1 < parent.Cursor)
                        {
                            var delta = value.Length - text.Length;
                            cursorMove = parent.Cursor + delta;
                        }
                    }

                    text = value;
                    if (cursorMove != null)
                        parent.Cursor = (int) cursorMove;
                }
            }

            public int? Cursor
            {
                get
                {
                    int lengthBefore = this.TextBefore().Length;
                    if (parent.Cursor < lengthBefore)
                        return null;

                    int offset = parent.Cursor - lengthBefore;

                    if (offset == Text.Length && Next?.IsHidden == false)
                    {
                        return null;
                    }

                    if (offset <= Text.Length)
                        return offset;
                    return null;
                }
                set
                {
                    if (value == null || value < 0 || value > Text.Length)
                        return;

                    parent.Cursor = this.TextBefore().Length + (int) value;
                }
            }

            public Token Next { get; set; }
            public Token Previous { get; set; }
            
            public IToken PreviousNotHidden => this.Previous?.ThisOrPrevIfHidden();
            public IToken NextNotHidden => this.Next?.ThisOrNextIfHidden();

            IToken IToken.Next => this.Next;

            IToken IToken.Previous => this.Previous;

            public Token First => this.Previous == null ? this : this.Previous.First;

            public Token FirstNonHidden => this.First.ThisOrNextIfHidden();

			public bool IsHidden { get; set; } = isHidden;

			public int? DistanceTo(IToken other, bool ignoreHidden = false)
            {
				if (other is not Token token)
					return null;

				var gap = Next?.ForwardTo(token, []);
                if (ignoreHidden)
                    return gap?.Where(x => !x.IsHidden).Count();
                else 
                    return gap?.Count;
            }

            private List<Token> ForwardTo(Token other, List<Token> gap)
            {
                gap.Add(this);

                if (other == this)
                    return gap;
                else
                {
                    return this.Next?.ForwardTo(other, gap);
                }
            }

            private Token[] TokensBefore()
            {
                var tokens = new List<Token>();
                var first = this.First;

                var pointer = first;
                while (pointer != this)
                {
                    tokens.Add(pointer);
                    pointer = pointer.Next;
                }

                return [.. tokens];
            }

            private string TextBefore()
            {
                return BuildText(this.TokensBefore());
            }

            private Token ThisOrNextIfHidden()
            {
                if (this.IsHidden)
                    return this.Next?.ThisOrNextIfHidden();
                else
                    return this;
            }

            private Token ThisOrPrevIfHidden()
            {
                if (this.IsHidden)
                    return this.Previous?.ThisOrPrevIfHidden();
                else
                    return this;
            }

            public static string BuildText(IEnumerable<Token> tokens)
            {
                var builder = new StringBuilder();
                foreach (var token in tokens)
                {
                    builder.Append(token.Text);
                }

                return builder.ToString();
            }
        }
    }
}