1 
2 //          Copyright Ferdinand Majerech 2011-2014.
3 // Distributed under the Boost Software License, Version 1.0.
4 //    (See accompanying file LICENSE_1_0.txt or copy at
5 //          http://www.boost.org/LICENSE_1_0.txt)
6 
7 /**
8  * YAML parser.
9  * Code based on PyYAML: http://www.pyyaml.org
10  */
11 module dyaml.parser;
12 
13 
14 import std.algorithm;
15 import std.array;
16 import std.conv;
17 import std.exception;
18 import std.typecons;
19 
20 import dyaml.event;
21 import dyaml.exception;
22 import dyaml.scanner;
23 import dyaml.style;
24 import dyaml.token;
25 import dyaml.tagdirective;
26 
27 
28 package:
29 /**
30  * The following YAML grammar is LL(1) and is parsed by a recursive descent
31  * parser.
32  *
33  * stream            ::= STREAM-START implicit_document? explicit_document* STREAM-END
34  * implicit_document ::= block_node DOCUMENT-END*
35  * explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END*
36  * block_node_or_indentless_sequence ::=
37  *                       ALIAS
38  *                       | properties (block_content | indentless_block_sequence)?
39  *                       | block_content
40  *                       | indentless_block_sequence
41  * block_node        ::= ALIAS
42  *                       | properties block_content?
43  *                       | block_content
44  * flow_node         ::= ALIAS
45  *                       | properties flow_content?
46  *                       | flow_content
47  * properties        ::= TAG ANCHOR? | ANCHOR TAG?
48  * block_content     ::= block_collection | flow_collection | SCALAR
49  * flow_content      ::= flow_collection | SCALAR
50  * block_collection  ::= block_sequence | block_mapping
51  * flow_collection   ::= flow_sequence | flow_mapping
52  * block_sequence    ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END
53  * indentless_sequence   ::= (BLOCK-ENTRY block_node?)+
54  * block_mapping     ::= BLOCK-MAPPING_START
55  *                       ((KEY block_node_or_indentless_sequence?)?
56  *                       (VALUE block_node_or_indentless_sequence?)?)*
57  *                       BLOCK-END
58  * flow_sequence     ::= FLOW-SEQUENCE-START
59  *                       (flow_sequence_entry FLOW-ENTRY)*
60  *                       flow_sequence_entry?
61  *                       FLOW-SEQUENCE-END
62  * flow_sequence_entry   ::= flow_node | KEY flow_node? (VALUE flow_node?)?
63  * flow_mapping      ::= FLOW-MAPPING-START
64  *                       (flow_mapping_entry FLOW-ENTRY)*
65  *                       flow_mapping_entry?
66  *                       FLOW-MAPPING-END
67  * flow_mapping_entry    ::= flow_node | KEY flow_node? (VALUE flow_node?)?
68  *
69  * FIRST sets:
70  *
71  * stream: { STREAM-START }
72  * explicit_document: { DIRECTIVE DOCUMENT-START }
73  * implicit_document: FIRST(block_node)
74  * block_node: { ALIAS TAG ANCHOR SCALAR BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START }
75  * flow_node: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START }
76  * block_content: { BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START SCALAR }
77  * flow_content: { FLOW-SEQUENCE-START FLOW-MAPPING-START SCALAR }
78  * block_collection: { BLOCK-SEQUENCE-START BLOCK-MAPPING-START }
79  * flow_collection: { FLOW-SEQUENCE-START FLOW-MAPPING-START }
80  * block_sequence: { BLOCK-SEQUENCE-START }
81  * block_mapping: { BLOCK-MAPPING-START }
82  * block_node_or_indentless_sequence: { ALIAS ANCHOR TAG SCALAR BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START BLOCK-ENTRY }
83  * indentless_sequence: { ENTRY }
84  * flow_collection: { FLOW-SEQUENCE-START FLOW-MAPPING-START }
85  * flow_sequence: { FLOW-SEQUENCE-START }
86  * flow_mapping: { FLOW-MAPPING-START }
87  * flow_sequence_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START KEY }
88  * flow_mapping_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START KEY }
89  */
90 
91 
92 /**
93  * Marked exception thrown at parser errors.
94  *
95  * See_Also: MarkedYAMLException
96  */
97 class ParserException : MarkedYAMLException
98 {
99     mixin MarkedExceptionCtors;
100 }
101 
102 /// Generates events from tokens provided by a Scanner.
103 ///
104 /// While Parser receives tokens with non-const character slices, the events it
105 /// produces are immutable strings, which are usually the same slices, cast to string.
106 /// Parser is the last layer of D:YAML that may possibly do any modifications to these
107 /// slices.
108 final class Parser
109 {
110     private:
111         ///Default tag handle shortcuts and replacements.
112         static TagDirective[] defaultTagDirectives_ =
113             [TagDirective("!", "!"), TagDirective("!!", "tag:yaml.org,2002:")];
114 
115         ///Scanner providing YAML tokens.
116         Scanner scanner_;
117 
118         ///Event produced by the most recent state.
119         Event currentEvent_;
120 
121         ///YAML version string.
122         string YAMLVersion_ = null;
123         ///Tag handle shortcuts and replacements.
124         TagDirective[] tagDirectives_;
125 
126         ///Stack of states.
127         Appender!(Event delegate() @safe[]) states_;
128         ///Stack of marks used to keep track of extents of e.g. YAML collections.
129         Appender!(Mark[]) marks_;
130 
131         ///Current state.
132         Event delegate() @safe state_;
133 
134     public:
135         ///Construct a Parser using specified Scanner.
136         this(Scanner scanner) @safe
137         {
138             state_ = &parseStreamStart;
139             scanner_ = scanner;
140             states_.reserve(32);
141             marks_.reserve(32);
142         }
143 
144         /**
145          * Check if any events are left. May have side effects in some cases.
146          */
147         bool empty() @safe
148         {
149             ensureState();
150             return currentEvent_.isNull;
151         }
152 
153         /**
154          * Return the current event.
155          *
156          * Must not be called if there are no events left.
157          */
158         Event front() @safe
159         {
160             ensureState();
161             assert(!currentEvent_.isNull, "No event left to peek");
162             return currentEvent_;
163         }
164 
165         /**
166          * Skip to the next event.
167          *
168          * Must not be called if there are no events left.
169          */
170         void popFront() @safe
171         {
172             currentEvent_.id = EventID.Invalid;
173             ensureState();
174         }
175 
176     private:
177         /// If current event is invalid, load the next valid one if possible.
178         void ensureState() @safe
179         {
180             if(currentEvent_.isNull && state_ !is null)
181             {
182                 currentEvent_ = state_();
183             }
184         }
185         ///Pop and return the newest state in states_.
186         Event delegate() @safe popState() @safe
187         {
188             enforce(states_.data.length > 0,
189                     new YAMLException("Parser: Need to pop state but no states left to pop"));
190             const result = states_.data.back;
191             states_.shrinkTo(states_.data.length - 1);
192             return result;
193         }
194 
195         ///Pop and return the newest mark in marks_.
196         Mark popMark() @safe
197         {
198             enforce(marks_.data.length > 0,
199                     new YAMLException("Parser: Need to pop mark but no marks left to pop"));
200             const result = marks_.data.back;
201             marks_.shrinkTo(marks_.data.length - 1);
202             return result;
203         }
204 
205         /// Push a state on the stack
206         void pushState(Event delegate() @safe state) @safe
207         {
208             states_ ~= state;
209         }
210         /// Push a mark on the stack
211         void pushMark(Mark mark) @safe
212         {
213             marks_ ~= mark;
214         }
215 
216         /**
217          * stream    ::= STREAM-START implicit_document? explicit_document* STREAM-END
218          * implicit_document ::= block_node DOCUMENT-END*
219          * explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END*
220          */
221 
222         ///Parse stream start.
223         Event parseStreamStart() @safe
224         {
225             const token = scanner_.getToken();
226             state_ = &parseImplicitDocumentStart;
227             return streamStartEvent(token.startMark, token.endMark);
228         }
229 
230         /// Parse implicit document start, unless explicit detected: if so, parse explicit.
231         Event parseImplicitDocumentStart() @safe
232         {
233             // Parse an implicit document.
234             if(!scanner_.checkToken(TokenID.Directive, TokenID.DocumentStart,
235                                     TokenID.StreamEnd))
236             {
237                 tagDirectives_  = defaultTagDirectives_;
238                 const token = scanner_.peekToken();
239 
240                 pushState(&parseDocumentEnd);
241                 state_ = &parseBlockNode;
242 
243                 return documentStartEvent(token.startMark, token.endMark, false, null, null);
244             }
245             return parseDocumentStart();
246         }
247 
248         ///Parse explicit document start.
249         Event parseDocumentStart() @trusted
250         {
251             //Parse any extra document end indicators.
252             while(scanner_.checkToken(TokenID.DocumentEnd)){scanner_.getToken();}
253 
254             //Parse an explicit document.
255             if(!scanner_.checkToken(TokenID.StreamEnd))
256             {
257                 const startMark = scanner_.peekToken().startMark;
258 
259                 auto tagDirectives = processDirectives();
260                 enforce(scanner_.checkToken(TokenID.DocumentStart),
261                         new ParserException("Expected document start but found " ~
262                                   scanner_.peekToken().idString,
263                                   scanner_.peekToken().startMark));
264 
265                 const endMark = scanner_.getToken().endMark;
266                 pushState(&parseDocumentEnd);
267                 state_ = &parseDocumentContent;
268                 return documentStartEvent(startMark, endMark, true, YAMLVersion_, tagDirectives);
269             }
270             else
271             {
272                 //Parse the end of the stream.
273                 const token = scanner_.getToken();
274                 assert(states_.data.length == 0);
275                 assert(marks_.data.length == 0);
276                 state_ = null;
277                 return streamEndEvent(token.startMark, token.endMark);
278             }
279         }
280 
281         ///Parse document end (explicit or implicit).
282         Event parseDocumentEnd() @safe
283         {
284             Mark startMark = scanner_.peekToken().startMark;
285             const bool explicit = scanner_.checkToken(TokenID.DocumentEnd);
286             Mark endMark = explicit ? scanner_.getToken().endMark : startMark;
287 
288             state_ = &parseDocumentStart;
289 
290             return documentEndEvent(startMark, endMark, explicit);
291         }
292 
293         ///Parse document content.
294         Event parseDocumentContent() @safe
295         {
296             if(scanner_.checkToken(TokenID.Directive,   TokenID.DocumentStart,
297                                    TokenID.DocumentEnd, TokenID.StreamEnd))
298             {
299                 state_ = popState();
300                 return processEmptyScalar(scanner_.peekToken().startMark);
301             }
302             return parseBlockNode();
303         }
304 
305         /// Process directives at the beginning of a document.
306         TagDirective[] processDirectives() @safe
307         {
308             // Destroy version and tag handles from previous document.
309             YAMLVersion_ = null;
310             tagDirectives_.length = 0;
311 
312             // Process directives.
313             while(scanner_.checkToken(TokenID.Directive))
314             {
315                 const token = scanner_.getToken();
316                 string value = token.value.idup;
317                 if(token.directive == DirectiveType.YAML)
318                 {
319                     enforce(YAMLVersion_ is null,
320                             new ParserException("Duplicate YAML directive", token.startMark));
321                     const minor = value.split(".")[0];
322                     enforce(minor == "1",
323                             new ParserException("Incompatible document (version 1.x is required)",
324                                       token.startMark));
325                     YAMLVersion_ = value;
326                 }
327                 else if(token.directive == DirectiveType.TAG)
328                 {
329                     auto handle = value[0 .. token.valueDivider];
330 
331                     foreach(ref pair; tagDirectives_)
332                     {
333                         // handle
334                         const h = pair.handle;
335                         enforce(h != handle, new ParserException("Duplicate tag handle: " ~ handle,
336                                                        token.startMark));
337                     }
338                     tagDirectives_ ~=
339                         TagDirective(handle, value[token.valueDivider .. $]);
340                 }
341                 // Any other directive type is ignored (only YAML and TAG are in YAML
342                 // 1.1/1.2, any other directives are "reserved")
343             }
344 
345             TagDirective[] value = tagDirectives_;
346 
347             //Add any default tag handles that haven't been overridden.
348             foreach(ref defaultPair; defaultTagDirectives_)
349             {
350                 bool found;
351                 foreach(ref pair; tagDirectives_) if(defaultPair.handle == pair.handle)
352                 {
353                     found = true;
354                     break;
355                 }
356                 if(!found) {tagDirectives_ ~= defaultPair; }
357             }
358 
359             return value;
360         }
361 
362         /**
363          * block_node_or_indentless_sequence ::= ALIAS
364          *               | properties (block_content | indentless_block_sequence)?
365          *               | block_content
366          *               | indentless_block_sequence
367          * block_node    ::= ALIAS
368          *                   | properties block_content?
369          *                   | block_content
370          * flow_node     ::= ALIAS
371          *                   | properties flow_content?
372          *                   | flow_content
373          * properties    ::= TAG ANCHOR? | ANCHOR TAG?
374          * block_content     ::= block_collection | flow_collection | SCALAR
375          * flow_content      ::= flow_collection | SCALAR
376          * block_collection  ::= block_sequence | block_mapping
377          * flow_collection   ::= flow_sequence | flow_mapping
378          */
379 
380         ///Parse a node.
381         Event parseNode(const Flag!"block" block,
382                         const Flag!"indentlessSequence" indentlessSequence = No.indentlessSequence)
383             @trusted
384         {
385             if(scanner_.checkToken(TokenID.Alias))
386             {
387                 const token = scanner_.getToken();
388                 state_ = popState();
389                 return aliasEvent(token.startMark, token.endMark,
390                                   cast(string)token.value);
391             }
392 
393             string anchor;
394             string tag;
395             Mark startMark, endMark, tagMark;
396             bool invalidMarks = true;
397             // The index in the tag string where tag handle ends and tag suffix starts.
398             uint tagHandleEnd;
399 
400             //Get anchor/tag if detected. Return false otherwise.
401             bool get(const TokenID id, const Flag!"first" first, ref string target) @safe
402             {
403                 if(!scanner_.checkToken(id)){return false;}
404                 invalidMarks = false;
405                 const token = scanner_.getToken();
406                 if(first){startMark = token.startMark;}
407                 if(id == TokenID.Tag)
408                 {
409                     tagMark = token.startMark;
410                     tagHandleEnd = token.valueDivider;
411                 }
412                 endMark = token.endMark;
413                 target  = token.value.idup;
414                 return true;
415             }
416 
417             //Anchor and/or tag can be in any order.
418             if(get(TokenID.Anchor, Yes.first, anchor)){get(TokenID.Tag, No.first, tag);}
419             else if(get(TokenID.Tag, Yes.first, tag)) {get(TokenID.Anchor, No.first, anchor);}
420 
421             if(tag !is null){tag = processTag(tag, tagHandleEnd, startMark, tagMark);}
422 
423             if(invalidMarks)
424             {
425                 startMark = endMark = scanner_.peekToken().startMark;
426             }
427 
428             bool implicit = (tag is null || tag == "!");
429 
430             if(indentlessSequence && scanner_.checkToken(TokenID.BlockEntry))
431             {
432                 state_ = &parseIndentlessSequenceEntry;
433                 return sequenceStartEvent
434                     (startMark, scanner_.peekToken().endMark, anchor,
435                      tag, implicit, CollectionStyle.Block);
436             }
437 
438             if(scanner_.checkToken(TokenID.Scalar))
439             {
440                 auto token = scanner_.getToken();
441                 auto value = token.style == ScalarStyle.DoubleQuoted
442                            ? handleDoubleQuotedScalarEscapes(token.value)
443                            : cast(string)token.value;
444 
445                 implicit = (token.style == ScalarStyle.Plain && tag is null) || tag == "!";
446                 state_ = popState();
447                 return scalarEvent(startMark, token.endMark, anchor, tag,
448                                    implicit, value, token.style);
449             }
450 
451             if(scanner_.checkToken(TokenID.FlowSequenceStart))
452             {
453                 endMark = scanner_.peekToken().endMark;
454                 state_ = &parseFlowSequenceEntry!(Yes.first);
455                 return sequenceStartEvent(startMark, endMark, anchor, tag,
456                                           implicit, CollectionStyle.Flow);
457             }
458 
459             if(scanner_.checkToken(TokenID.FlowMappingStart))
460             {
461                 endMark = scanner_.peekToken().endMark;
462                 state_ = &parseFlowMappingKey!(Yes.first);
463                 return mappingStartEvent(startMark, endMark, anchor, tag,
464                                          implicit, CollectionStyle.Flow);
465             }
466 
467             if(block && scanner_.checkToken(TokenID.BlockSequenceStart))
468             {
469                 endMark = scanner_.peekToken().endMark;
470                 state_ = &parseBlockSequenceEntry!(Yes.first);
471                 return sequenceStartEvent(startMark, endMark, anchor, tag,
472                                           implicit, CollectionStyle.Block);
473             }
474 
475             if(block && scanner_.checkToken(TokenID.BlockMappingStart))
476             {
477                 endMark = scanner_.peekToken().endMark;
478                 state_ = &parseBlockMappingKey!(Yes.first);
479                 return mappingStartEvent(startMark, endMark, anchor, tag,
480                                          implicit, CollectionStyle.Block);
481             }
482 
483             if(anchor !is null || tag !is null)
484             {
485                 state_ = popState();
486 
487                 //PyYAML uses a tuple(implicit, false) for the second last arg here,
488                 //but the second bool is never used after that - so we don't use it.
489 
490                 //Empty scalars are allowed even if a tag or an anchor is specified.
491                 return scalarEvent(startMark, endMark, anchor, tag,
492                                    implicit , "");
493             }
494 
495             const token = scanner_.peekToken();
496             throw new ParserException("While parsing a " ~ (block ? "block" : "flow") ~ " node",
497                             startMark, "expected node content, but found: "
498                             ~ token.idString, token.startMark);
499         }
500 
501         /// Handle escape sequences in a double quoted scalar.
502         ///
503         /// Moved here from scanner as it can't always be done in-place with slices.
504         string handleDoubleQuotedScalarEscapes(char[] tokenValue) const @safe
505         {
506             string notInPlace;
507             bool inEscape;
508             auto appender = appender!(string)();
509             for(char[] oldValue = tokenValue; !oldValue.empty();)
510             {
511                 const dchar c = oldValue.front();
512                 oldValue.popFront();
513 
514                 if(!inEscape)
515                 {
516                     if(c != '\\')
517                     {
518                         if(notInPlace is null) { appender.put(c); }
519                         else                   { notInPlace ~= c; }
520                         continue;
521                     }
522                     // Escape sequence starts with a '\'
523                     inEscape = true;
524                     continue;
525                 }
526 
527                 import dyaml.escapes;
528                 scope(exit) { inEscape = false; }
529 
530                 // 'Normal' escape sequence.
531                 if(dyaml.escapes.escapes.canFind(c))
532                 {
533                     if(notInPlace is null)
534                     {
535                         // \L and \C can't be handled in place as the expand into
536                         // many-byte unicode chars
537                         if(c != 'L' && c != 'P')
538                         {
539                             appender.put(dyaml.escapes.fromEscape(c));
540                             continue;
541                         }
542                         // Need to duplicate as we won't fit into
543                         // token.value - which is what appender uses
544                         notInPlace = appender.data.dup;
545                         notInPlace ~= dyaml.escapes.fromEscape(c);
546                         continue;
547                     }
548                     notInPlace ~= dyaml.escapes.fromEscape(c);
549                     continue;
550                 }
551 
552                 // Unicode char written in hexadecimal in an escape sequence.
553                 if(dyaml.escapes.escapeHexCodeList.canFind(c))
554                 {
555                     // Scanner has already checked that the hex string is valid.
556 
557                     const hexLength = dyaml.escapes.escapeHexLength(c);
558                     // Any hex digits are 1-byte so this works.
559                     char[] hex = oldValue[0 .. hexLength];
560                     oldValue = oldValue[hexLength .. $];
561                     import std.ascii : isHexDigit;
562                     assert(!hex.canFind!(d => !d.isHexDigit),
563                             "Scanner must ensure the hex string is valid");
564 
565                     const decoded = cast(dchar)parse!int(hex, 16u);
566                     if(notInPlace is null) { appender.put(decoded); }
567                     else                   { notInPlace ~= decoded; }
568                     continue;
569                 }
570 
571                 assert(false, "Scanner must handle unsupported escapes");
572             }
573 
574             return notInPlace is null ? appender.data : notInPlace;
575         }
576 
577         /**
578          * Process a tag string retrieved from a tag token.
579          *
580          * Params:  tag       = Tag before processing.
581          *          handleEnd = Index in tag where tag handle ends and tag suffix
582          *                      starts.
583          *          startMark = Position of the node the tag belongs to.
584          *          tagMark   = Position of the tag.
585          */
586         string processTag(const string tag, const uint handleEnd,
587                           const Mark startMark, const Mark tagMark)
588             const @safe
589         {
590             const handle = tag[0 .. handleEnd];
591             const suffix = tag[handleEnd .. $];
592 
593             if(handle.length > 0)
594             {
595                 string replacement;
596                 foreach(ref pair; tagDirectives_)
597                 {
598                     if(pair.handle == handle)
599                     {
600                         replacement = pair.prefix;
601                         break;
602                     }
603                 }
604                 //handle must be in tagDirectives_
605                 enforce(replacement !is null,
606                         new ParserException("While parsing a node", startMark,
607                                   "found undefined tag handle: " ~ handle, tagMark));
608                 return replacement ~ suffix;
609             }
610             return suffix;
611         }
612 
613         ///Wrappers to parse nodes.
614         Event parseBlockNode() @safe {return parseNode(Yes.block);}
615         Event parseFlowNode() @safe {return parseNode(No.block);}
616         Event parseBlockNodeOrIndentlessSequence() @safe {return parseNode(Yes.block, Yes.indentlessSequence);}
617 
618         ///block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END
619 
620         ///Parse an entry of a block sequence. If first is true, this is the first entry.
621         Event parseBlockSequenceEntry(Flag!"first" first)() @safe
622         {
623             static if(first){pushMark(scanner_.getToken().startMark);}
624 
625             if(scanner_.checkToken(TokenID.BlockEntry))
626             {
627                 const token = scanner_.getToken();
628                 if(!scanner_.checkToken(TokenID.BlockEntry, TokenID.BlockEnd))
629                 {
630                     pushState(&parseBlockSequenceEntry!(No.first));
631                     return parseBlockNode();
632                 }
633 
634                 state_ = &parseBlockSequenceEntry!(No.first);
635                 return processEmptyScalar(token.endMark);
636             }
637 
638             if(!scanner_.checkToken(TokenID.BlockEnd))
639             {
640                 const token = scanner_.peekToken();
641                 throw new ParserException("While parsing a block collection", marks_.data.back,
642                                 "expected block end, but found " ~ token.idString,
643                                 token.startMark);
644             }
645 
646             state_ = popState();
647             popMark();
648             const token = scanner_.getToken();
649             return sequenceEndEvent(token.startMark, token.endMark);
650         }
651 
652         ///indentless_sequence ::= (BLOCK-ENTRY block_node?)+
653 
654         ///Parse an entry of an indentless sequence.
655         Event parseIndentlessSequenceEntry() @safe
656         {
657             if(scanner_.checkToken(TokenID.BlockEntry))
658             {
659                 const token = scanner_.getToken();
660 
661                 if(!scanner_.checkToken(TokenID.BlockEntry, TokenID.Key,
662                                         TokenID.Value, TokenID.BlockEnd))
663                 {
664                     pushState(&parseIndentlessSequenceEntry);
665                     return parseBlockNode();
666                 }
667 
668                 state_ = &parseIndentlessSequenceEntry;
669                 return processEmptyScalar(token.endMark);
670             }
671 
672             state_ = popState();
673             const token = scanner_.peekToken();
674             return sequenceEndEvent(token.startMark, token.endMark);
675         }
676 
677         /**
678          * block_mapping     ::= BLOCK-MAPPING_START
679          *                       ((KEY block_node_or_indentless_sequence?)?
680          *                       (VALUE block_node_or_indentless_sequence?)?)*
681          *                       BLOCK-END
682          */
683 
684         ///Parse a key in a block mapping. If first is true, this is the first key.
685         Event parseBlockMappingKey(Flag!"first" first)() @safe
686         {
687             static if(first){pushMark(scanner_.getToken().startMark);}
688 
689             if(scanner_.checkToken(TokenID.Key))
690             {
691                 const token = scanner_.getToken();
692 
693                 if(!scanner_.checkToken(TokenID.Key, TokenID.Value, TokenID.BlockEnd))
694                 {
695                     pushState(&parseBlockMappingValue);
696                     return parseBlockNodeOrIndentlessSequence();
697                 }
698 
699                 state_ = &parseBlockMappingValue;
700                 return processEmptyScalar(token.endMark);
701             }
702 
703             if(!scanner_.checkToken(TokenID.BlockEnd))
704             {
705                 const token = scanner_.peekToken();
706                 throw new ParserException("While parsing a block mapping", marks_.data.back,
707                                 "expected block end, but found: " ~ token.idString,
708                                 token.startMark);
709             }
710 
711             state_ = popState();
712             popMark();
713             const token = scanner_.getToken();
714             return mappingEndEvent(token.startMark, token.endMark);
715         }
716 
717         ///Parse a value in a block mapping.
718         Event parseBlockMappingValue() @safe
719         {
720             if(scanner_.checkToken(TokenID.Value))
721             {
722                 const token = scanner_.getToken();
723 
724                 if(!scanner_.checkToken(TokenID.Key, TokenID.Value, TokenID.BlockEnd))
725                 {
726                     pushState(&parseBlockMappingKey!(No.first));
727                     return parseBlockNodeOrIndentlessSequence();
728                 }
729 
730                 state_ = &parseBlockMappingKey!(No.first);
731                 return processEmptyScalar(token.endMark);
732             }
733 
734             state_= &parseBlockMappingKey!(No.first);
735             return processEmptyScalar(scanner_.peekToken().startMark);
736         }
737 
738         /**
739          * flow_sequence     ::= FLOW-SEQUENCE-START
740          *                       (flow_sequence_entry FLOW-ENTRY)*
741          *                       flow_sequence_entry?
742          *                       FLOW-SEQUENCE-END
743          * flow_sequence_entry   ::= flow_node | KEY flow_node? (VALUE flow_node?)?
744          *
745          * Note that while production rules for both flow_sequence_entry and
746          * flow_mapping_entry are equal, their interpretations are different.
747          * For `flow_sequence_entry`, the part `KEY flow_node? (VALUE flow_node?)?`
748          * generate an inline mapping (set syntax).
749          */
750 
751         ///Parse an entry in a flow sequence. If first is true, this is the first entry.
752         Event parseFlowSequenceEntry(Flag!"first" first)() @safe
753         {
754             static if(first){pushMark(scanner_.getToken().startMark);}
755 
756             if(!scanner_.checkToken(TokenID.FlowSequenceEnd))
757             {
758                 static if(!first)
759                 {
760                     if(scanner_.checkToken(TokenID.FlowEntry))
761                     {
762                         scanner_.getToken();
763                     }
764                     else
765                     {
766                         const token = scanner_.peekToken();
767                         throw new ParserException("While parsing a flow sequence", marks_.data.back,
768                                         "expected ',' or ']', but got: " ~
769                                         token.idString, token.startMark);
770                     }
771                 }
772 
773                 if(scanner_.checkToken(TokenID.Key))
774                 {
775                     const token = scanner_.peekToken();
776                     state_ = &parseFlowSequenceEntryMappingKey;
777                     return mappingStartEvent(token.startMark, token.endMark,
778                                              null, null, true, CollectionStyle.Flow);
779                 }
780                 else if(!scanner_.checkToken(TokenID.FlowSequenceEnd))
781                 {
782                     pushState(&parseFlowSequenceEntry!(No.first));
783                     return parseFlowNode();
784                 }
785             }
786 
787             const token = scanner_.getToken();
788             state_ = popState();
789             popMark();
790             return sequenceEndEvent(token.startMark, token.endMark);
791         }
792 
793         ///Parse a key in flow context.
794         Event parseFlowKey(in Event delegate() @safe nextState) @safe
795         {
796             const token = scanner_.getToken();
797 
798             if(!scanner_.checkToken(TokenID.Value, TokenID.FlowEntry,
799                                     TokenID.FlowSequenceEnd))
800             {
801                 pushState(nextState);
802                 return parseFlowNode();
803             }
804 
805             state_ = nextState;
806             return processEmptyScalar(token.endMark);
807         }
808 
809         ///Parse a mapping key in an entry in a flow sequence.
810         Event parseFlowSequenceEntryMappingKey() @safe
811         {
812             return parseFlowKey(&parseFlowSequenceEntryMappingValue);
813         }
814 
815         ///Parse a mapping value in a flow context.
816         Event parseFlowValue(TokenID checkId, in Event delegate() @safe nextState)
817             @safe
818         {
819             if(scanner_.checkToken(TokenID.Value))
820             {
821                 const token = scanner_.getToken();
822                 if(!scanner_.checkToken(TokenID.FlowEntry, checkId))
823                 {
824                     pushState(nextState);
825                     return parseFlowNode();
826                 }
827 
828                 state_ = nextState;
829                 return processEmptyScalar(token.endMark);
830             }
831 
832             state_ = nextState;
833             return processEmptyScalar(scanner_.peekToken().startMark);
834         }
835 
836         ///Parse a mapping value in an entry in a flow sequence.
837         Event parseFlowSequenceEntryMappingValue() @safe
838         {
839             return parseFlowValue(TokenID.FlowSequenceEnd,
840                                   &parseFlowSequenceEntryMappingEnd);
841         }
842 
843         ///Parse end of a mapping in a flow sequence entry.
844         Event parseFlowSequenceEntryMappingEnd() @safe
845         {
846             state_ = &parseFlowSequenceEntry!(No.first);
847             const token = scanner_.peekToken();
848             return mappingEndEvent(token.startMark, token.startMark);
849         }
850 
851         /**
852          * flow_mapping  ::= FLOW-MAPPING-START
853          *                   (flow_mapping_entry FLOW-ENTRY)*
854          *                   flow_mapping_entry?
855          *                   FLOW-MAPPING-END
856          * flow_mapping_entry    ::= flow_node | KEY flow_node? (VALUE flow_node?)?
857          */
858 
859         ///Parse a key in a flow mapping.
860         Event parseFlowMappingKey(Flag!"first" first)() @safe
861         {
862             static if(first){pushMark(scanner_.getToken().startMark);}
863 
864             if(!scanner_.checkToken(TokenID.FlowMappingEnd))
865             {
866                 static if(!first)
867                 {
868                     if(scanner_.checkToken(TokenID.FlowEntry))
869                     {
870                         scanner_.getToken();
871                     }
872                     else
873                     {
874                         const token = scanner_.peekToken();
875                         throw new ParserException("While parsing a flow mapping", marks_.data.back,
876                                         "expected ',' or '}', but got: " ~
877                                         token.idString, token.startMark);
878                     }
879                 }
880 
881                 if(scanner_.checkToken(TokenID.Key))
882                 {
883                     return parseFlowKey(&parseFlowMappingValue);
884                 }
885 
886                 if(!scanner_.checkToken(TokenID.FlowMappingEnd))
887                 {
888                     pushState(&parseFlowMappingEmptyValue);
889                     return parseFlowNode();
890                 }
891             }
892 
893             const token = scanner_.getToken();
894             state_ = popState();
895             popMark();
896             return mappingEndEvent(token.startMark, token.endMark);
897         }
898 
899         ///Parse a value in a flow mapping.
900         Event parseFlowMappingValue()  @safe
901         {
902             return parseFlowValue(TokenID.FlowMappingEnd, &parseFlowMappingKey!(No.first));
903         }
904 
905         ///Parse an empty value in a flow mapping.
906         Event parseFlowMappingEmptyValue() @safe
907         {
908             state_ = &parseFlowMappingKey!(No.first);
909             return processEmptyScalar(scanner_.peekToken().startMark);
910         }
911 
912         ///Return an empty scalar.
913         Event processEmptyScalar(const Mark mark) @safe pure nothrow const @nogc
914         {
915             return scalarEvent(mark, mark, null, null, true, "");
916         }
917 }