1 
2 //          Copyright Ferdinand Majerech 2011-2014.
3 // Distributed under the Boost Software License, Version 1.0.
4 //    (See accompanying file LICENSE_1_0.txt or copy at
5 //          http://www.boost.org/LICENSE_1_0.txt)
6 
7 /**
8  * YAML parser.
9  * Code based on PyYAML: http://www.pyyaml.org
10  */
11 module dyaml.parser;
12 
13 
14 import std.algorithm;
15 import std.array;
16 import std.conv;
17 import std.exception;
18 import std.typecons;
19 
20 import dyaml.event;
21 import dyaml.exception;
22 import dyaml.scanner;
23 import dyaml.style;
24 import dyaml.token;
25 import dyaml.tagdirective;
26 
27 
28 package:
29 /**
30  * The following YAML grammar is LL(1) and is parsed by a recursive descent
31  * parser.
32  *
33  * stream            ::= STREAM-START implicit_document? explicit_document* STREAM-END
34  * implicit_document ::= block_node DOCUMENT-END*
35  * explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END*
36  * block_node_or_indentless_sequence ::=
37  *                       ALIAS
38  *                       | properties (block_content | indentless_block_sequence)?
39  *                       | block_content
40  *                       | indentless_block_sequence
41  * block_node        ::= ALIAS
42  *                       | properties block_content?
43  *                       | block_content
44  * flow_node         ::= ALIAS
45  *                       | properties flow_content?
46  *                       | flow_content
47  * properties        ::= TAG ANCHOR? | ANCHOR TAG?
48  * block_content     ::= block_collection | flow_collection | SCALAR
49  * flow_content      ::= flow_collection | SCALAR
50  * block_collection  ::= block_sequence | block_mapping
51  * flow_collection   ::= flow_sequence | flow_mapping
52  * block_sequence    ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END
53  * indentless_sequence   ::= (BLOCK-ENTRY block_node?)+
54  * block_mapping     ::= BLOCK-MAPPING_START
55  *                       ((KEY block_node_or_indentless_sequence?)?
56  *                       (VALUE block_node_or_indentless_sequence?)?)*
57  *                       BLOCK-END
58  * flow_sequence     ::= FLOW-SEQUENCE-START
59  *                       (flow_sequence_entry FLOW-ENTRY)*
60  *                       flow_sequence_entry?
61  *                       FLOW-SEQUENCE-END
62  * flow_sequence_entry   ::= flow_node | KEY flow_node? (VALUE flow_node?)?
63  * flow_mapping      ::= FLOW-MAPPING-START
64  *                       (flow_mapping_entry FLOW-ENTRY)*
65  *                       flow_mapping_entry?
66  *                       FLOW-MAPPING-END
67  * flow_mapping_entry    ::= flow_node | KEY flow_node? (VALUE flow_node?)?
68  *
69  * FIRST sets:
70  *
71  * stream: { STREAM-START }
72  * explicit_document: { DIRECTIVE DOCUMENT-START }
73  * implicit_document: FIRST(block_node)
74  * block_node: { ALIAS TAG ANCHOR SCALAR BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START }
75  * flow_node: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START }
76  * block_content: { BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START SCALAR }
77  * flow_content: { FLOW-SEQUENCE-START FLOW-MAPPING-START SCALAR }
78  * block_collection: { BLOCK-SEQUENCE-START BLOCK-MAPPING-START }
79  * flow_collection: { FLOW-SEQUENCE-START FLOW-MAPPING-START }
80  * block_sequence: { BLOCK-SEQUENCE-START }
81  * block_mapping: { BLOCK-MAPPING-START }
82  * block_node_or_indentless_sequence: { ALIAS ANCHOR TAG SCALAR BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START BLOCK-ENTRY }
83  * indentless_sequence: { ENTRY }
84  * flow_collection: { FLOW-SEQUENCE-START FLOW-MAPPING-START }
85  * flow_sequence: { FLOW-SEQUENCE-START }
86  * flow_mapping: { FLOW-MAPPING-START }
87  * flow_sequence_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START KEY }
88  * flow_mapping_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START KEY }
89  */
90 
91 
92 /**
93  * Marked exception thrown at parser errors.
94  *
95  * See_Also: MarkedYAMLException
96  */
97 class ParserException : MarkedYAMLException
98 {
99     mixin MarkedExceptionCtors;
100 }
101 
102 /// Generates events from tokens provided by a Scanner.
103 ///
104 /// While Parser receives tokens with non-const character slices, the events it
105 /// produces are immutable strings, which are usually the same slices, cast to string.
106 /// Parser is the last layer of D:YAML that may possibly do any modifications to these
107 /// slices.
108 final class Parser
109 {
110     private:
111         ///Default tag handle shortcuts and replacements.
112         static TagDirective[] defaultTagDirectives_ =
113             [TagDirective("!", "!"), TagDirective("!!", "tag:yaml.org,2002:")];
114 
115         ///Scanner providing YAML tokens.
116         Scanner scanner_;
117 
118         ///Event produced by the most recent state.
119         Event currentEvent_;
120 
121         ///YAML version string.
122         string YAMLVersion_ = null;
123         ///Tag handle shortcuts and replacements.
124         TagDirective[] tagDirectives_;
125 
126         ///Stack of states.
127         Appender!(Event delegate() @safe[]) states_;
128         ///Stack of marks used to keep track of extents of e.g. YAML collections.
129         Appender!(Mark[]) marks_;
130 
131         ///Current state.
132         Event delegate() @safe state_;
133 
134     public:
135         ///Construct a Parser using specified Scanner.
136         this(Scanner scanner) @safe
137         {
138             state_ = &parseStreamStart;
139             scanner_ = scanner;
140             states_.reserve(32);
141             marks_.reserve(32);
142         }
143 
144         /**
145          * Check if any events are left. May have side effects in some cases.
146          */
147         bool empty() @safe
148         {
149             ensureState();
150             return currentEvent_.isNull;
151         }
152 
153         /**
154          * Return the current event.
155          *
156          * Must not be called if there are no events left.
157          */
158         Event front() @safe
159         {
160             ensureState();
161             assert(!currentEvent_.isNull, "No event left to peek");
162             return currentEvent_;
163         }
164 
165         /**
166          * Skip to the next event.
167          *
168          * Must not be called if there are no events left.
169          */
170         void popFront() @safe
171         {
172             currentEvent_.id = EventID.invalid;
173             ensureState();
174         }
175 
176     private:
177         /// If current event is invalid, load the next valid one if possible.
178         void ensureState() @safe
179         {
180             if(currentEvent_.isNull && state_ !is null)
181             {
182                 currentEvent_ = state_();
183             }
184         }
185         ///Pop and return the newest state in states_.
186         Event delegate() @safe popState() @safe
187         {
188             enforce(states_.data.length > 0,
189                     new YAMLException("Parser: Need to pop state but no states left to pop"));
190             const result = states_.data.back;
191             states_.shrinkTo(states_.data.length - 1);
192             return result;
193         }
194 
195         ///Pop and return the newest mark in marks_.
196         Mark popMark() @safe
197         {
198             enforce(marks_.data.length > 0,
199                     new YAMLException("Parser: Need to pop mark but no marks left to pop"));
200             const result = marks_.data.back;
201             marks_.shrinkTo(marks_.data.length - 1);
202             return result;
203         }
204 
205         /// Push a state on the stack
206         void pushState(Event delegate() @safe state) @safe
207         {
208             states_ ~= state;
209         }
210         /// Push a mark on the stack
211         void pushMark(Mark mark) @safe
212         {
213             marks_ ~= mark;
214         }
215 
216         /**
217          * stream    ::= STREAM-START implicit_document? explicit_document* STREAM-END
218          * implicit_document ::= block_node DOCUMENT-END*
219          * explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END*
220          */
221 
222         ///Parse stream start.
223         Event parseStreamStart() @safe
224         {
225             const token = scanner_.front;
226             scanner_.popFront();
227             state_ = &parseImplicitDocumentStart;
228             return streamStartEvent(token.startMark, token.endMark);
229         }
230 
231         /// Parse implicit document start, unless explicit detected: if so, parse explicit.
232         Event parseImplicitDocumentStart() @safe
233         {
234             // Parse an implicit document.
235             if(!scanner_.front.id.among!(TokenID.directive, TokenID.documentStart,
236                                     TokenID.streamEnd))
237             {
238                 tagDirectives_  = defaultTagDirectives_;
239                 const token = scanner_.front;
240 
241                 pushState(&parseDocumentEnd);
242                 state_ = &parseBlockNode;
243 
244                 return documentStartEvent(token.startMark, token.endMark, false, null, null);
245             }
246             return parseDocumentStart();
247         }
248 
249         ///Parse explicit document start.
250         Event parseDocumentStart() @safe
251         {
252             //Parse any extra document end indicators.
253             while(scanner_.front.id == TokenID.documentEnd)
254             {
255                 scanner_.popFront();
256             }
257 
258             //Parse an explicit document.
259             if(scanner_.front.id != TokenID.streamEnd)
260             {
261                 const startMark = scanner_.front.startMark;
262 
263                 auto tagDirectives = processDirectives();
264                 enforce(scanner_.front.id == TokenID.documentStart,
265                         new ParserException("Expected document start but found " ~
266                                   scanner_.front.idString,
267                                   scanner_.front.startMark));
268 
269                 const endMark = scanner_.front.endMark;
270                 scanner_.popFront();
271                 pushState(&parseDocumentEnd);
272                 state_ = &parseDocumentContent;
273                 return documentStartEvent(startMark, endMark, true, YAMLVersion_, tagDirectives);
274             }
275             else
276             {
277                 //Parse the end of the stream.
278                 const token = scanner_.front;
279                 scanner_.popFront();
280                 assert(states_.data.length == 0);
281                 assert(marks_.data.length == 0);
282                 state_ = null;
283                 return streamEndEvent(token.startMark, token.endMark);
284             }
285         }
286 
287         ///Parse document end (explicit or implicit).
288         Event parseDocumentEnd() @safe
289         {
290             Mark startMark = scanner_.front.startMark;
291             const bool explicit = scanner_.front.id == TokenID.documentEnd;
292             Mark endMark = startMark;
293             if (explicit)
294             {
295                 endMark = scanner_.front.endMark;
296                 scanner_.popFront();
297             }
298 
299             state_ = &parseDocumentStart;
300 
301             return documentEndEvent(startMark, endMark, explicit);
302         }
303 
304         ///Parse document content.
305         Event parseDocumentContent() @safe
306         {
307             if(scanner_.front.id.among!(TokenID.directive,   TokenID.documentStart,
308                                    TokenID.documentEnd, TokenID.streamEnd))
309             {
310                 state_ = popState();
311                 return processEmptyScalar(scanner_.front.startMark);
312             }
313             return parseBlockNode();
314         }
315 
316         /// Process directives at the beginning of a document.
317         TagDirective[] processDirectives() @safe
318         {
319             // Destroy version and tag handles from previous document.
320             YAMLVersion_ = null;
321             tagDirectives_.length = 0;
322 
323             // Process directives.
324             while(scanner_.front.id == TokenID.directive)
325             {
326                 const token = scanner_.front;
327                 scanner_.popFront();
328                 string value = token.value.idup;
329                 if(token.directive == DirectiveType.yaml)
330                 {
331                     enforce(YAMLVersion_ is null,
332                             new ParserException("Duplicate YAML directive", token.startMark));
333                     const minor = value.split(".")[0];
334                     enforce(minor == "1",
335                             new ParserException("Incompatible document (version 1.x is required)",
336                                       token.startMark));
337                     YAMLVersion_ = value;
338                 }
339                 else if(token.directive == DirectiveType.tag)
340                 {
341                     auto handle = value[0 .. token.valueDivider];
342 
343                     foreach(ref pair; tagDirectives_)
344                     {
345                         // handle
346                         const h = pair.handle;
347                         enforce(h != handle, new ParserException("Duplicate tag handle: " ~ handle,
348                                                        token.startMark));
349                     }
350                     tagDirectives_ ~=
351                         TagDirective(handle, value[token.valueDivider .. $]);
352                 }
353                 // Any other directive type is ignored (only YAML and TAG are in YAML
354                 // 1.1/1.2, any other directives are "reserved")
355             }
356 
357             TagDirective[] value = tagDirectives_;
358 
359             //Add any default tag handles that haven't been overridden.
360             foreach(ref defaultPair; defaultTagDirectives_)
361             {
362                 bool found;
363                 foreach(ref pair; tagDirectives_) if(defaultPair.handle == pair.handle)
364                 {
365                     found = true;
366                     break;
367                 }
368                 if(!found) {tagDirectives_ ~= defaultPair; }
369             }
370 
371             return value;
372         }
373 
374         /**
375          * block_node_or_indentless_sequence ::= ALIAS
376          *               | properties (block_content | indentless_block_sequence)?
377          *               | block_content
378          *               | indentless_block_sequence
379          * block_node    ::= ALIAS
380          *                   | properties block_content?
381          *                   | block_content
382          * flow_node     ::= ALIAS
383          *                   | properties flow_content?
384          *                   | flow_content
385          * properties    ::= TAG ANCHOR? | ANCHOR TAG?
386          * block_content     ::= block_collection | flow_collection | SCALAR
387          * flow_content      ::= flow_collection | SCALAR
388          * block_collection  ::= block_sequence | block_mapping
389          * flow_collection   ::= flow_sequence | flow_mapping
390          */
391 
392         ///Parse a node.
393         Event parseNode(const Flag!"block" block,
394                         const Flag!"indentlessSequence" indentlessSequence = No.indentlessSequence)
395             @trusted
396         {
397             if(scanner_.front.id == TokenID.alias_)
398             {
399                 const token = scanner_.front;
400                 scanner_.popFront();
401                 state_ = popState();
402                 return aliasEvent(token.startMark, token.endMark,
403                                   cast(string)token.value);
404             }
405 
406             string anchor;
407             string tag;
408             Mark startMark, endMark, tagMark;
409             bool invalidMarks = true;
410             // The index in the tag string where tag handle ends and tag suffix starts.
411             uint tagHandleEnd;
412 
413             //Get anchor/tag if detected. Return false otherwise.
414             bool get(const TokenID id, const Flag!"first" first, ref string target) @safe
415             {
416                 if(scanner_.front.id != id){return false;}
417                 invalidMarks = false;
418                 const token = scanner_.front;
419                 scanner_.popFront();
420                 if(first){startMark = token.startMark;}
421                 if(id == TokenID.tag)
422                 {
423                     tagMark = token.startMark;
424                     tagHandleEnd = token.valueDivider;
425                 }
426                 endMark = token.endMark;
427                 target  = token.value.idup;
428                 return true;
429             }
430 
431             //Anchor and/or tag can be in any order.
432             if(get(TokenID.anchor, Yes.first, anchor)){get(TokenID.tag, No.first, tag);}
433             else if(get(TokenID.tag, Yes.first, tag)) {get(TokenID.anchor, No.first, anchor);}
434 
435             if(tag !is null){tag = processTag(tag, tagHandleEnd, startMark, tagMark);}
436 
437             if(invalidMarks)
438             {
439                 startMark = endMark = scanner_.front.startMark;
440             }
441 
442             bool implicit = (tag is null || tag == "!");
443 
444             if(indentlessSequence && scanner_.front.id == TokenID.blockEntry)
445             {
446                 state_ = &parseIndentlessSequenceEntry;
447                 return sequenceStartEvent
448                     (startMark, scanner_.front.endMark, anchor,
449                      tag, implicit, CollectionStyle.block);
450             }
451 
452             if(scanner_.front.id == TokenID.scalar)
453             {
454                 auto token = scanner_.front;
455                 scanner_.popFront();
456                 auto value = token.style == ScalarStyle.doubleQuoted
457                            ? handleDoubleQuotedScalarEscapes(token.value)
458                            : cast(string)token.value;
459 
460                 implicit = (token.style == ScalarStyle.plain && tag is null) || tag == "!";
461                 state_ = popState();
462                 return scalarEvent(startMark, token.endMark, anchor, tag,
463                                    implicit, value, token.style);
464             }
465 
466             if(scanner_.front.id == TokenID.flowSequenceStart)
467             {
468                 endMark = scanner_.front.endMark;
469                 state_ = &parseFlowSequenceEntry!(Yes.first);
470                 return sequenceStartEvent(startMark, endMark, anchor, tag,
471                                           implicit, CollectionStyle.flow);
472             }
473 
474             if(scanner_.front.id == TokenID.flowMappingStart)
475             {
476                 endMark = scanner_.front.endMark;
477                 state_ = &parseFlowMappingKey!(Yes.first);
478                 return mappingStartEvent(startMark, endMark, anchor, tag,
479                                          implicit, CollectionStyle.flow);
480             }
481 
482             if(block && scanner_.front.id == TokenID.blockSequenceStart)
483             {
484                 endMark = scanner_.front.endMark;
485                 state_ = &parseBlockSequenceEntry!(Yes.first);
486                 return sequenceStartEvent(startMark, endMark, anchor, tag,
487                                           implicit, CollectionStyle.block);
488             }
489 
490             if(block && scanner_.front.id == TokenID.blockMappingStart)
491             {
492                 endMark = scanner_.front.endMark;
493                 state_ = &parseBlockMappingKey!(Yes.first);
494                 return mappingStartEvent(startMark, endMark, anchor, tag,
495                                          implicit, CollectionStyle.block);
496             }
497 
498             if(anchor !is null || tag !is null)
499             {
500                 state_ = popState();
501 
502                 //PyYAML uses a tuple(implicit, false) for the second last arg here,
503                 //but the second bool is never used after that - so we don't use it.
504 
505                 //Empty scalars are allowed even if a tag or an anchor is specified.
506                 return scalarEvent(startMark, endMark, anchor, tag,
507                                    implicit , "");
508             }
509 
510             const token = scanner_.front;
511             throw new ParserException("While parsing a " ~ (block ? "block" : "flow") ~ " node",
512                             startMark, "expected node content, but found: "
513                             ~ token.idString, token.startMark);
514         }
515 
516         /// Handle escape sequences in a double quoted scalar.
517         ///
518         /// Moved here from scanner as it can't always be done in-place with slices.
519         string handleDoubleQuotedScalarEscapes(const(char)[] tokenValue) const @safe
520         {
521             string notInPlace;
522             bool inEscape;
523             auto appender = appender!(string)();
524             for(const(char)[] oldValue = tokenValue; !oldValue.empty();)
525             {
526                 const dchar c = oldValue.front();
527                 oldValue.popFront();
528 
529                 if(!inEscape)
530                 {
531                     if(c != '\\')
532                     {
533                         if(notInPlace is null) { appender.put(c); }
534                         else                   { notInPlace ~= c; }
535                         continue;
536                     }
537                     // Escape sequence starts with a '\'
538                     inEscape = true;
539                     continue;
540                 }
541 
542                 import dyaml.escapes;
543                 scope(exit) { inEscape = false; }
544 
545                 // 'Normal' escape sequence.
546                 if(c.among!(escapes))
547                 {
548                     if(notInPlace is null)
549                     {
550                         // \L and \C can't be handled in place as the expand into
551                         // many-byte unicode chars
552                         if(c != 'L' && c != 'P')
553                         {
554                             appender.put(dyaml.escapes.fromEscape(c));
555                             continue;
556                         }
557                         // Need to duplicate as we won't fit into
558                         // token.value - which is what appender uses
559                         notInPlace = appender.data.dup;
560                         notInPlace ~= dyaml.escapes.fromEscape(c);
561                         continue;
562                     }
563                     notInPlace ~= dyaml.escapes.fromEscape(c);
564                     continue;
565                 }
566 
567                 // Unicode char written in hexadecimal in an escape sequence.
568                 if(c.among!(escapeHexCodeList))
569                 {
570                     // Scanner has already checked that the hex string is valid.
571 
572                     const hexLength = dyaml.escapes.escapeHexLength(c);
573                     // Any hex digits are 1-byte so this works.
574                     const(char)[] hex = oldValue[0 .. hexLength];
575                     oldValue = oldValue[hexLength .. $];
576                     import std.ascii : isHexDigit;
577                     assert(!hex.canFind!(d => !d.isHexDigit),
578                             "Scanner must ensure the hex string is valid");
579 
580                     const decoded = cast(dchar)parse!int(hex, 16u);
581                     if(notInPlace is null) { appender.put(decoded); }
582                     else                   { notInPlace ~= decoded; }
583                     continue;
584                 }
585 
586                 assert(false, "Scanner must handle unsupported escapes");
587             }
588 
589             return notInPlace is null ? appender.data : notInPlace;
590         }
591 
592         /**
593          * Process a tag string retrieved from a tag token.
594          *
595          * Params:  tag       = Tag before processing.
596          *          handleEnd = Index in tag where tag handle ends and tag suffix
597          *                      starts.
598          *          startMark = Position of the node the tag belongs to.
599          *          tagMark   = Position of the tag.
600          */
601         string processTag(const string tag, const uint handleEnd,
602                           const Mark startMark, const Mark tagMark)
603             const @safe
604         {
605             const handle = tag[0 .. handleEnd];
606             const suffix = tag[handleEnd .. $];
607 
608             if(handle.length > 0)
609             {
610                 string replacement;
611                 foreach(ref pair; tagDirectives_)
612                 {
613                     if(pair.handle == handle)
614                     {
615                         replacement = pair.prefix;
616                         break;
617                     }
618                 }
619                 //handle must be in tagDirectives_
620                 enforce(replacement !is null,
621                         new ParserException("While parsing a node", startMark,
622                                   "found undefined tag handle: " ~ handle, tagMark));
623                 return replacement ~ suffix;
624             }
625             return suffix;
626         }
627 
628         ///Wrappers to parse nodes.
629         Event parseBlockNode() @safe {return parseNode(Yes.block);}
630         Event parseFlowNode() @safe {return parseNode(No.block);}
631         Event parseBlockNodeOrIndentlessSequence() @safe {return parseNode(Yes.block, Yes.indentlessSequence);}
632 
633         ///block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END
634 
635         ///Parse an entry of a block sequence. If first is true, this is the first entry.
636         Event parseBlockSequenceEntry(Flag!"first" first)() @safe
637         {
638             static if(first)
639             {
640                 pushMark(scanner_.front.startMark);
641                 scanner_.popFront();
642             }
643 
644             if(scanner_.front.id == TokenID.blockEntry)
645             {
646                 const token = scanner_.front;
647                 scanner_.popFront();
648                 if(!scanner_.front.id.among!(TokenID.blockEntry, TokenID.blockEnd))
649                 {
650                     pushState(&parseBlockSequenceEntry!(No.first));
651                     return parseBlockNode();
652                 }
653 
654                 state_ = &parseBlockSequenceEntry!(No.first);
655                 return processEmptyScalar(token.endMark);
656             }
657 
658             if(scanner_.front.id != TokenID.blockEnd)
659             {
660                 const token = scanner_.front;
661                 throw new ParserException("While parsing a block collection", marks_.data.back,
662                                 "expected block end, but found " ~ token.idString,
663                                 token.startMark);
664             }
665 
666             state_ = popState();
667             popMark();
668             const token = scanner_.front;
669             scanner_.popFront();
670             return sequenceEndEvent(token.startMark, token.endMark);
671         }
672 
673         ///indentless_sequence ::= (BLOCK-ENTRY block_node?)+
674 
675         ///Parse an entry of an indentless sequence.
676         Event parseIndentlessSequenceEntry() @safe
677         {
678             if(scanner_.front.id == TokenID.blockEntry)
679             {
680                 const token = scanner_.front;
681                 scanner_.popFront();
682 
683                 if(!scanner_.front.id.among!(TokenID.blockEntry, TokenID.key,
684                                         TokenID.value, TokenID.blockEnd))
685                 {
686                     pushState(&parseIndentlessSequenceEntry);
687                     return parseBlockNode();
688                 }
689 
690                 state_ = &parseIndentlessSequenceEntry;
691                 return processEmptyScalar(token.endMark);
692             }
693 
694             state_ = popState();
695             const token = scanner_.front;
696             return sequenceEndEvent(token.startMark, token.endMark);
697         }
698 
699         /**
700          * block_mapping     ::= BLOCK-MAPPING_START
701          *                       ((KEY block_node_or_indentless_sequence?)?
702          *                       (VALUE block_node_or_indentless_sequence?)?)*
703          *                       BLOCK-END
704          */
705 
706         ///Parse a key in a block mapping. If first is true, this is the first key.
707         Event parseBlockMappingKey(Flag!"first" first)() @safe
708         {
709             static if(first)
710             {
711                 pushMark(scanner_.front.startMark);
712                 scanner_.popFront();
713             }
714 
715             if(scanner_.front.id == TokenID.key)
716             {
717                 const token = scanner_.front;
718                 scanner_.popFront();
719 
720                 if(!scanner_.front.id.among!(TokenID.key, TokenID.value, TokenID.blockEnd))
721                 {
722                     pushState(&parseBlockMappingValue);
723                     return parseBlockNodeOrIndentlessSequence();
724                 }
725 
726                 state_ = &parseBlockMappingValue;
727                 return processEmptyScalar(token.endMark);
728             }
729 
730             if(scanner_.front.id != TokenID.blockEnd)
731             {
732                 const token = scanner_.front;
733                 throw new ParserException("While parsing a block mapping", marks_.data.back,
734                                 "expected block end, but found: " ~ token.idString,
735                                 token.startMark);
736             }
737 
738             state_ = popState();
739             popMark();
740             const token = scanner_.front;
741             scanner_.popFront();
742             return mappingEndEvent(token.startMark, token.endMark);
743         }
744 
745         ///Parse a value in a block mapping.
746         Event parseBlockMappingValue() @safe
747         {
748             if(scanner_.front.id == TokenID.value)
749             {
750                 const token = scanner_.front;
751                 scanner_.popFront();
752 
753                 if(!scanner_.front.id.among!(TokenID.key, TokenID.value, TokenID.blockEnd))
754                 {
755                     pushState(&parseBlockMappingKey!(No.first));
756                     return parseBlockNodeOrIndentlessSequence();
757                 }
758 
759                 state_ = &parseBlockMappingKey!(No.first);
760                 return processEmptyScalar(token.endMark);
761             }
762 
763             state_= &parseBlockMappingKey!(No.first);
764             return processEmptyScalar(scanner_.front.startMark);
765         }
766 
767         /**
768          * flow_sequence     ::= FLOW-SEQUENCE-START
769          *                       (flow_sequence_entry FLOW-ENTRY)*
770          *                       flow_sequence_entry?
771          *                       FLOW-SEQUENCE-END
772          * flow_sequence_entry   ::= flow_node | KEY flow_node? (VALUE flow_node?)?
773          *
774          * Note that while production rules for both flow_sequence_entry and
775          * flow_mapping_entry are equal, their interpretations are different.
776          * For `flow_sequence_entry`, the part `KEY flow_node? (VALUE flow_node?)?`
777          * generate an inline mapping (set syntax).
778          */
779 
780         ///Parse an entry in a flow sequence. If first is true, this is the first entry.
781         Event parseFlowSequenceEntry(Flag!"first" first)() @safe
782         {
783             static if(first)
784             {
785                 pushMark(scanner_.front.startMark);
786                 scanner_.popFront();
787             }
788 
789             if(scanner_.front.id != TokenID.flowSequenceEnd)
790             {
791                 static if(!first)
792                 {
793                     if(scanner_.front.id == TokenID.flowEntry)
794                     {
795                         scanner_.popFront();
796                     }
797                     else
798                     {
799                         const token = scanner_.front;
800                         throw new ParserException("While parsing a flow sequence", marks_.data.back,
801                                         "expected ',' or ']', but got: " ~
802                                         token.idString, token.startMark);
803                     }
804                 }
805 
806                 if(scanner_.front.id == TokenID.key)
807                 {
808                     const token = scanner_.front;
809                     state_ = &parseFlowSequenceEntryMappingKey;
810                     return mappingStartEvent(token.startMark, token.endMark,
811                                              null, null, true, CollectionStyle.flow);
812                 }
813                 else if(scanner_.front.id != TokenID.flowSequenceEnd)
814                 {
815                     pushState(&parseFlowSequenceEntry!(No.first));
816                     return parseFlowNode();
817                 }
818             }
819 
820             const token = scanner_.front;
821             scanner_.popFront();
822             state_ = popState();
823             popMark();
824             return sequenceEndEvent(token.startMark, token.endMark);
825         }
826 
827         ///Parse a key in flow context.
828         Event parseFlowKey(Event delegate() @safe nextState) @safe
829         {
830             const token = scanner_.front;
831             scanner_.popFront();
832 
833             if(!scanner_.front.id.among!(TokenID.value, TokenID.flowEntry,
834                                     TokenID.flowSequenceEnd))
835             {
836                 pushState(nextState);
837                 return parseFlowNode();
838             }
839 
840             state_ = nextState;
841             return processEmptyScalar(token.endMark);
842         }
843 
844         ///Parse a mapping key in an entry in a flow sequence.
845         Event parseFlowSequenceEntryMappingKey() @safe
846         {
847             return parseFlowKey(&parseFlowSequenceEntryMappingValue);
848         }
849 
850         ///Parse a mapping value in a flow context.
851         Event parseFlowValue(TokenID checkId, Event delegate() @safe nextState)
852             @safe
853         {
854             if(scanner_.front.id == TokenID.value)
855             {
856                 const token = scanner_.front;
857                 scanner_.popFront();
858                 if(!scanner_.front.id.among(TokenID.flowEntry, checkId))
859                 {
860                     pushState(nextState);
861                     return parseFlowNode();
862                 }
863 
864                 state_ = nextState;
865                 return processEmptyScalar(token.endMark);
866             }
867 
868             state_ = nextState;
869             return processEmptyScalar(scanner_.front.startMark);
870         }
871 
872         ///Parse a mapping value in an entry in a flow sequence.
873         Event parseFlowSequenceEntryMappingValue() @safe
874         {
875             return parseFlowValue(TokenID.flowSequenceEnd,
876                                   &parseFlowSequenceEntryMappingEnd);
877         }
878 
879         ///Parse end of a mapping in a flow sequence entry.
880         Event parseFlowSequenceEntryMappingEnd() @safe
881         {
882             state_ = &parseFlowSequenceEntry!(No.first);
883             const token = scanner_.front;
884             return mappingEndEvent(token.startMark, token.startMark);
885         }
886 
887         /**
888          * flow_mapping  ::= FLOW-MAPPING-START
889          *                   (flow_mapping_entry FLOW-ENTRY)*
890          *                   flow_mapping_entry?
891          *                   FLOW-MAPPING-END
892          * flow_mapping_entry    ::= flow_node | KEY flow_node? (VALUE flow_node?)?
893          */
894 
895         ///Parse a key in a flow mapping.
896         Event parseFlowMappingKey(Flag!"first" first)() @safe
897         {
898             static if(first)
899             {
900                 pushMark(scanner_.front.startMark);
901                 scanner_.popFront();
902             }
903 
904             if(scanner_.front.id != TokenID.flowMappingEnd)
905             {
906                 static if(!first)
907                 {
908                     if(scanner_.front.id == TokenID.flowEntry)
909                     {
910                         scanner_.popFront();
911                     }
912                     else
913                     {
914                         const token = scanner_.front;
915                         throw new ParserException("While parsing a flow mapping", marks_.data.back,
916                                         "expected ',' or '}', but got: " ~
917                                         token.idString, token.startMark);
918                     }
919                 }
920 
921                 if(scanner_.front.id == TokenID.key)
922                 {
923                     return parseFlowKey(&parseFlowMappingValue);
924                 }
925 
926                 if(scanner_.front.id != TokenID.flowMappingEnd)
927                 {
928                     pushState(&parseFlowMappingEmptyValue);
929                     return parseFlowNode();
930                 }
931             }
932 
933             const token = scanner_.front;
934             scanner_.popFront();
935             state_ = popState();
936             popMark();
937             return mappingEndEvent(token.startMark, token.endMark);
938         }
939 
940         ///Parse a value in a flow mapping.
941         Event parseFlowMappingValue()  @safe
942         {
943             return parseFlowValue(TokenID.flowMappingEnd, &parseFlowMappingKey!(No.first));
944         }
945 
946         ///Parse an empty value in a flow mapping.
947         Event parseFlowMappingEmptyValue() @safe
948         {
949             state_ = &parseFlowMappingKey!(No.first);
950             return processEmptyScalar(scanner_.front.startMark);
951         }
952 
953         ///Return an empty scalar.
954         Event processEmptyScalar(const Mark mark) @safe pure nothrow const @nogc
955         {
956             return scalarEvent(mark, mark, null, null, true, "");
957         }
958 }