1 
2 //          Copyright Ferdinand Majerech 2011-2014.
3 // Distributed under the Boost Software License, Version 1.0.
4 //    (See accompanying file LICENSE_1_0.txt or copy at
5 //          http://www.boost.org/LICENSE_1_0.txt)
6 
7 /// YAML tokens.
8 /// Code based on PyYAML: http://www.pyyaml.org
9 module dyaml.token;
10 
11 
12 import std.conv;
13 
14 import dyaml.encoding;
15 import dyaml.exception;
16 import dyaml.reader;
17 import dyaml.style;
18 
19 
20 package:
21 
22 /// Token types.
23 enum TokenID : ubyte
24 {
25     Invalid = 0,        /// Invalid (uninitialized) token
26     Directive,          /// DIRECTIVE
27     DocumentStart,      /// DOCUMENT-START
28     DocumentEnd,        /// DOCUMENT-END
29     StreamStart,        /// STREAM-START
30     StreamEnd,          /// STREAM-END
31     BlockSequenceStart, /// BLOCK-SEQUENCE-START
32     BlockMappingStart,  /// BLOCK-MAPPING-START
33     BlockEnd,           /// BLOCK-END
34     FlowSequenceStart,  /// FLOW-SEQUENCE-START
35     FlowMappingStart,   /// FLOW-MAPPING-START
36     FlowSequenceEnd,    /// FLOW-SEQUENCE-END
37     FlowMappingEnd,     /// FLOW-MAPPING-END
38     Key,                /// KEY
39     Value,              /// VALUE
40     BlockEntry,         /// BLOCK-ENTRY
41     FlowEntry,          /// FLOW-ENTRY
42     Alias,              /// ALIAS
43     Anchor,             /// ANCHOR
44     Tag,                /// TAG
45     Scalar              /// SCALAR
46 }
47 
48 /// Specifies the type of a tag directive token.
49 enum DirectiveType : ubyte
50 {
51     // YAML version directive.
52     YAML,
53     // Tag directive.
54     TAG,
55     // Any other directive is "reserved" for future YAML versions.
56     Reserved
57 }
58 
59 /// Token produced by scanner.
60 ///
61 /// 32 bytes on 64-bit.
62 struct Token
63 {
64     @disable int opCmp(ref Token);
65 
66     // 16B
67     /// Value of the token, if any.
68     ///
69     /// Values are char[] instead of string, as Parser may still change them in a few
70     /// cases. Parser casts values to strings when producing Events.
71     char[] value;
72     // 4B
73     /// Start position of the token in file/stream.
74     Mark startMark;
75     // 4B
76     /// End position of the token in file/stream.
77     Mark endMark;
78     // 1B
79     /// Token type.
80     TokenID id;
81     // 1B
82     /// Style of scalar token, if this is a scalar token.
83     ScalarStyle style;
84     // 1B
85     /// Encoding, if this is a stream start token.
86     Encoding encoding;
87     // 1B
88     /// Type of directive for directiveToken.
89     DirectiveType directive;
90     // 4B
91     /// Used to split value into 2 substrings for tokens that need 2 values (tagToken)
92     uint valueDivider;
93 
94     /// Get string representation of the token ID.
95     @property string idString() @safe pure const {return id.to!string;}
96 }
97 static assert(Token.sizeof <= 32, "Token has unexpected size");
98 
99 
100 /// Construct a directive token.
101 ///
102 /// Params:  start     = Start position of the token.
103 ///          end       = End position of the token.
104 ///          value     = Value of the token.
105 ///          directive = Directive type (YAML or TAG in YAML 1.1).
106 ///          nameEnd = Position of the end of the name
107 Token directiveToken(const Mark start, const Mark end, char[] value,
108                      DirectiveType directive, const uint nameEnd) @safe pure nothrow @nogc
109 {
110     return Token(value, start, end, TokenID.Directive, ScalarStyle.init, Encoding.init,
111                  directive, nameEnd);
112 }
113 
114 /// Construct a simple (no value) token with specified type.
115 ///
116 /// Params:  id    = Type of the token.
117 ///          start = Start position of the token.
118 ///          end   = End position of the token.
119 Token simpleToken(TokenID id)(const Mark start, const Mark end)
120 {
121     return Token(null, start, end, id);
122 }
123 
124 /// Construct a stream start token.
125 ///
126 /// Params:  start    = Start position of the token.
127 ///          end      = End position of the token.
128 ///          encoding = Encoding of the stream.
129 Token streamStartToken(const Mark start, const Mark end, const Encoding encoding) @safe pure nothrow @nogc
130 {
131     return Token(null, start, end, TokenID.StreamStart, ScalarStyle.Invalid, encoding);
132 }
133 
134 /// Aliases for construction of simple token types.
135 alias streamEndToken = simpleToken!(TokenID.StreamEnd);
136 alias blockSequenceStartToken = simpleToken!(TokenID.BlockSequenceStart);
137 alias blockMappingStartToken = simpleToken!(TokenID.BlockMappingStart);
138 alias blockEndToken = simpleToken!(TokenID.BlockEnd);
139 alias keyToken = simpleToken!(TokenID.Key);
140 alias valueToken = simpleToken!(TokenID.Value);
141 alias blockEntryToken = simpleToken!(TokenID.BlockEntry);
142 alias flowEntryToken = simpleToken!(TokenID.FlowEntry);
143 
144 /// Construct a simple token with value with specified type.
145 ///
146 /// Params:  id           = Type of the token.
147 ///          start        = Start position of the token.
148 ///          end          = End position of the token.
149 ///          value        = Value of the token.
150 ///          valueDivider = A hack for TagToken to store 2 values in value; the first
151 ///                         value goes up to valueDivider, the second after it.
152 Token simpleValueToken(TokenID id)(const Mark start, const Mark end, char[] value,
153                                    const uint valueDivider = uint.max)
154 {
155     return Token(value, start, end, id, ScalarStyle.Invalid, Encoding.init,
156                  DirectiveType.init, valueDivider);
157 }
158 
159 /// Alias for construction of tag token.
160 alias tagToken = simpleValueToken!(TokenID.Tag);
161 alias aliasToken = simpleValueToken!(TokenID.Alias);
162 alias anchorToken = simpleValueToken!(TokenID.Anchor);
163 
164 /// Construct a scalar token.
165 ///
166 /// Params:  start = Start position of the token.
167 ///          end   = End position of the token.
168 ///          value = Value of the token.
169 ///          style = Style of the token.
170 Token scalarToken(const Mark start, const Mark end, char[] value, const ScalarStyle style) @safe pure nothrow @nogc
171 {
172     return Token(value, start, end, TokenID.Scalar, style);
173 }