1 
2 //          Copyright Ferdinand Majerech 2011.
3 // Distributed under the Boost Software License, Version 1.0.
4 //    (See accompanying file LICENSE_1_0.txt or copy at
5 //          http://www.boost.org/LICENSE_1_0.txt)
6 
7 /**
8  * YAML node _representer. Prepares YAML nodes for output. A tutorial can be
9  * found $(LINK2 ../tutorials/custom_types.html, here).
10  *
11  * Code based on $(LINK2 http://www.pyyaml.org, PyYAML).
12  */
13 module dyaml.representer;
14 
15 
16 import std.algorithm;
17 import std.array;
18 import std.base64;
19 import std.container;
20 import std.conv;
21 import std.datetime;
22 import std.exception;
23 import std.format;
24 import std.math;
25 import std.typecons;
26 import std..string;
27 
28 import dyaml.exception;
29 import dyaml.node;
30 import dyaml.serializer;
31 import dyaml.style;
32 
33 
34 ///Exception thrown on Representer errors.
35 class RepresenterException : YAMLException
36 {
37     mixin ExceptionCtors;
38 }
39 
40 /**
41  * Represents YAML nodes as scalar, sequence and mapping nodes ready for output.
42  *
43  * This class is used to add support for dumping of custom data types.
44  *
45  * It can also override default node formatting styles for output.
46  */
47 final class Representer
48 {
49     private:
50         // Representer functions indexed by types.
51         Node function(ref Node, Representer) @safe[TypeInfo] representers_;
52         // Default style for scalar nodes.
53         ScalarStyle defaultScalarStyle_ = ScalarStyle.Invalid;
54         // Default style for collection nodes.
55         CollectionStyle defaultCollectionStyle_ = CollectionStyle.Invalid;
56 
57     public:
58         @disable bool opEquals(ref Representer);
59         @disable int opCmp(ref Representer);
60 
61         /**
62          * Construct a Representer.
63          *
64          * Params:  useDefaultRepresenters = Use default representer functions
65          *                                   for default YAML types? This can be
66          *                                   disabled to use custom representer
67          *                                   functions for default types.
68          */
69         this(const Flag!"useDefaultRepresenters" useDefaultRepresenters = Yes.useDefaultRepresenters)
70             @safe pure
71         {
72             if(!useDefaultRepresenters){return;}
73             addRepresenter!YAMLNull(&representNull);
74             addRepresenter!string(&representString);
75             addRepresenter!(ubyte[])(&representBytes);
76             addRepresenter!bool(&representBool);
77             addRepresenter!long(&representLong);
78             addRepresenter!real(&representReal);
79             addRepresenter!(Node[])(&representNodes);
80             addRepresenter!(Node.Pair[])(&representPairs);
81             addRepresenter!SysTime(&representSysTime);
82         }
83 
84         ///Set default _style for scalars. If style is $(D ScalarStyle.Invalid), the _style is chosen automatically.
85         @property void defaultScalarStyle(ScalarStyle style) pure @safe nothrow
86         {
87             defaultScalarStyle_ = style;
88         }
89 
90         ///Set default _style for collections. If style is $(D CollectionStyle.Invalid), the _style is chosen automatically.
91         @property void defaultCollectionStyle(CollectionStyle style) pure @safe nothrow
92         {
93             defaultCollectionStyle_ = style;
94         }
95 
96         /**
97          * Add a function to represent nodes with a specific data type.
98          *
99          * The representer function takes references to a $(D Node) storing the data
100          * type and to the $(D Representer). It returns the represented node and may
101          * throw a $(D RepresenterException). See the example for more information.
102          *
103          *
104          * Only one function may be specified for one data type. Default data
105          * types already have representer functions unless disabled in the
106          * $(D Representer) constructor.
107          *
108          *
109          * Structs and classes must implement the $(D opCmp()) operator for D:YAML
110          * support. The signature of the operator that must be implemented
111          * is $(D const int opCmp(ref const MyStruct s)) for structs where
112          * $(I MyStruct) is the struct type, and $(D int opCmp(Object o)) for
113          * classes. Note that the class $(D opCmp()) should not alter the compared
114          * values - it is not const for compatibility reasons.
115          *
116          * Params:  representer = Representer function to add.
117          */
118         void addRepresenter(T)(Node function(ref Node, Representer) @safe representer)
119             @safe pure
120         {
121             assert((typeid(T) in representers_) is null,
122                    "Representer function for data type " ~ T.stringof ~
123                    " already specified. Can't specify another one");
124             representers_[typeid(T)] = representer;
125         }
126         /// Representing a simple struct:
127         unittest {
128             import std..string;
129 
130             import dyaml;
131 
132             struct MyStruct
133             {
134                 int x, y, z;
135 
136                 //Any D:YAML type must have a custom opCmp operator.
137                 //This is used for ordering in mappings.
138                 const int opCmp(ref const MyStruct s)
139                 {
140                     if(x != s.x){return x - s.x;}
141                     if(y != s.y){return y - s.y;}
142                     if(z != s.z){return z - s.z;}
143                     return 0;
144                 }
145             }
146 
147             static Node representMyStruct(ref Node node, Representer representer) @safe
148             {
149                 //The node is guaranteed to be MyStruct as we add representer for MyStruct.
150                 auto value = node.as!MyStruct;
151                 //Using custom scalar format, x:y:z.
152                 auto scalar = format("%s:%s:%s", value.x, value.y, value.z);
153                 //Representing as a scalar, with custom tag to specify this data type.
154                 return representer.representScalar("!mystruct.tag", scalar);
155             }
156 
157             auto dumper = dumper(new Appender!string);
158             auto representer = new Representer;
159             representer.addRepresenter!MyStruct(&representMyStruct);
160             dumper.representer = representer;
161             dumper.dump(Node(MyStruct(1,2,3)));
162         }
163         /// Representing a class:
164         unittest {
165             import std..string;
166 
167             import dyaml;
168 
169             class MyClass
170             {
171                 int x, y, z;
172 
173                 this(int x, int y, int z)
174                 {
175                     this.x = x;
176                     this.y = y;
177                     this.z = z;
178                 }
179 
180                 //Any D:YAML type must have a custom opCmp operator.
181                 //This is used for ordering in mappings.
182                 override int opCmp(Object o)
183                 {
184                     MyClass s = cast(MyClass)o;
185                     if(s is null){return -1;}
186                     if(x != s.x){return x - s.x;}
187                     if(y != s.y){return y - s.y;}
188                     if(z != s.z){return z - s.z;}
189                     return 0;
190                 }
191 
192                 ///Useful for Node.as!string .
193                 override string toString()
194                 {
195                     return format("MyClass(%s, %s, %s)", x, y, z);
196                 }
197             }
198 
199             //Same as representMyStruct.
200             static Node representMyClass(ref Node node, Representer representer) @safe
201             {
202                 //The node is guaranteed to be MyClass as we add representer for MyClass.
203                 auto value = node.as!MyClass;
204                 //Using custom scalar format, x:y:z.
205                 auto scalar = format("%s:%s:%s", value.x, value.y, value.z);
206                 //Representing as a scalar, with custom tag to specify this data type.
207                 return representer.representScalar("!myclass.tag", scalar);
208             }
209 
210             auto dumper = dumper(new Appender!string);
211             auto representer = new Representer;
212             representer.addRepresenter!MyClass(&representMyClass);
213             dumper.representer = representer;
214             dumper.dump(Node(new MyClass(1,2,3)));
215         }
216 
217         //If profiling shows a bottleneck on tag construction in these 3 methods,
218         //we'll need to take Tag directly and have string based wrappers for
219         //user code.
220 
221         /**
222          * Represent a _scalar with specified _tag.
223          *
224          * This is used by representer functions that produce scalars.
225          *
226          * Params:  tag    = Tag of the _scalar.
227          *          scalar = Scalar value.
228          *          style  = Style of the _scalar. If invalid, default _style will be used.
229          *                   If the node was loaded before, previous _style will always be used.
230          *
231          * Returns: The represented node.
232          */
233         Node representScalar(string tag, string scalar,
234                              ScalarStyle style = ScalarStyle.Invalid) @safe
235         {
236             if(style == ScalarStyle.Invalid){style = defaultScalarStyle_;}
237             auto newNode = Node(scalar, tag);
238             newNode.scalarStyle = style;
239             return newNode;
240         }
241         ///
242         @safe unittest
243         {
244             struct MyStruct
245             {
246                 int x, y, z;
247 
248                 //Any D:YAML type must have a custom opCmp operator.
249                 //This is used for ordering in mappings.
250                 const int opCmp(ref const MyStruct s)
251                 {
252                     if(x != s.x){return x - s.x;}
253                     if(y != s.y){return y - s.y;}
254                     if(z != s.z){return z - s.z;}
255                     return 0;
256                 }
257             }
258 
259             static Node representMyStruct(ref Node node, Representer representer)
260             {
261                 auto value = node.as!MyStruct;
262                 auto scalar = format("%s:%s:%s", value.x, value.y, value.z);
263                 return representer.representScalar("!mystruct.tag", scalar);
264             }
265 
266             auto dumper = dumper(new Appender!string);
267             auto representer = new Representer;
268             representer.addRepresenter!MyStruct(&representMyStruct);
269             dumper.representer = representer;
270             dumper.dump(Node(MyStruct(1,2,3)));
271         }
272 
273         /**
274          * Represent a _sequence with specified _tag, representing children first.
275          *
276          * This is used by representer functions that produce sequences.
277          *
278          * Params:  tag      = Tag of the _sequence.
279          *          sequence = Sequence of nodes.
280          *          style    = Style of the _sequence. If invalid, default _style will be used.
281          *                     If the node was loaded before, previous _style will always be used.
282          *
283          * Returns: The represented node.
284          *
285          * Throws:  $(D RepresenterException) if a child could not be represented.
286          */
287         Node representSequence(string tag, Node[] sequence,
288                                CollectionStyle style = CollectionStyle.Invalid) @safe
289         {
290             Node[] value;
291             value.length = sequence.length;
292 
293             auto bestStyle = CollectionStyle.Flow;
294             foreach(idx, ref item; sequence)
295             {
296                 value[idx] = representData(item);
297                 const isScalar = value[idx].isScalar;
298                 const s = value[idx].scalarStyle;
299                 if(!isScalar || (s != ScalarStyle.Invalid && s != ScalarStyle.Plain))
300                 {
301                     bestStyle = CollectionStyle.Block;
302                 }
303             }
304 
305             if(style == CollectionStyle.Invalid)
306             {
307                 style = defaultCollectionStyle_ != CollectionStyle.Invalid
308                         ? defaultCollectionStyle_
309                         : bestStyle;
310             }
311             auto newNode = Node(value, tag);
312             newNode.collectionStyle = style;
313             return newNode;
314         }
315         ///
316         @safe unittest
317         {
318             struct MyStruct
319             {
320                 int x, y, z;
321 
322                 //Any D:YAML type must have a custom opCmp operator.
323                 //This is used for ordering in mappings.
324                 const int opCmp(ref const MyStruct s)
325                 {
326                     if(x != s.x){return x - s.x;}
327                     if(y != s.y){return y - s.y;}
328                     if(z != s.z){return z - s.z;}
329                     return 0;
330                 }
331             }
332 
333             static Node representMyStruct(ref Node node, Representer representer)
334             {
335                 auto value = node.as!MyStruct;
336                 auto nodes = [Node(value.x), Node(value.y), Node(value.z)];
337                 //use flow style
338                 return representer.representSequence("!mystruct.tag", nodes,
339                     CollectionStyle.Flow);
340             }
341 
342             auto dumper = dumper(new Appender!string);
343             auto representer = new Representer;
344             representer.addRepresenter!MyStruct(&representMyStruct);
345             dumper.representer = representer;
346             dumper.dump(Node(MyStruct(1,2,3)));
347         }
348         /**
349          * Represent a mapping with specified _tag, representing children first.
350          *
351          * This is used by representer functions that produce mappings.
352          *
353          * Params:  tag   = Tag of the mapping.
354          *          pairs = Key-value _pairs of the mapping.
355          *          style = Style of the mapping. If invalid, default _style will be used.
356          *                  If the node was loaded before, previous _style will always be used.
357          *
358          * Returns: The represented node.
359          *
360          * Throws:  $(D RepresenterException) if a child could not be represented.
361          */
362         Node representMapping(string tag, Node.Pair[] pairs,
363                               CollectionStyle style = CollectionStyle.Invalid) @safe
364         {
365             Node.Pair[] value;
366             value.length = pairs.length;
367 
368             auto bestStyle = CollectionStyle.Flow;
369             foreach(idx, ref pair; pairs)
370             {
371                 value[idx] = Node.Pair(representData(pair.key), representData(pair.value));
372                 const keyScalar = value[idx].key.isScalar;
373                 const valScalar = value[idx].value.isScalar;
374                 const keyStyle = value[idx].key.scalarStyle;
375                 const valStyle = value[idx].value.scalarStyle;
376                 if(!keyScalar ||
377                    (keyStyle != ScalarStyle.Invalid && keyStyle != ScalarStyle.Plain))
378                 {
379                     bestStyle = CollectionStyle.Block;
380                 }
381                 if(!valScalar ||
382                    (valStyle != ScalarStyle.Invalid && valStyle != ScalarStyle.Plain))
383                 {
384                     bestStyle = CollectionStyle.Block;
385                 }
386             }
387 
388             if(style == CollectionStyle.Invalid)
389             {
390                 style = defaultCollectionStyle_ != CollectionStyle.Invalid
391                         ? defaultCollectionStyle_
392                         : bestStyle;
393             }
394             auto newNode = Node(value, tag);
395             newNode.collectionStyle = style;
396             return newNode;
397         }
398         ///
399         @safe unittest
400         {
401             struct MyStruct
402             {
403                 int x, y, z;
404 
405                 //Any D:YAML type must have a custom opCmp operator.
406                 //This is used for ordering in mappings.
407                 const int opCmp(ref const MyStruct s)
408                 {
409                     if(x != s.x){return x - s.x;}
410                     if(y != s.y){return y - s.y;}
411                     if(z != s.z){return z - s.z;}
412                     return 0;
413                 }
414             }
415 
416             static Node representMyStruct(ref Node node, Representer representer)
417             {
418                 auto value = node.as!MyStruct;
419                 auto pairs = [Node.Pair("x", value.x),
420                 Node.Pair("y", value.y),
421                 Node.Pair("z", value.z)];
422                 return representer.representMapping("!mystruct.tag", pairs);
423             }
424 
425             auto dumper = dumper(new Appender!string);
426             auto representer = new Representer;
427             representer.addRepresenter!MyStruct(&representMyStruct);
428             dumper.representer = representer;
429             dumper.dump(Node(MyStruct(1,2,3)));
430         }
431 
432     package:
433         //Represent a node based on its type, and return the represented result.
434         Node representData(ref Node data) @safe
435         {
436             //User types are wrapped in YAMLObject.
437             auto type = data.isUserType ? data.as!YAMLObject.type : data.type;
438 
439             enforce((type in representers_) !is null,
440                     new RepresenterException("No representer function for type "
441                                              ~ type.toString() ~ " , cannot represent."));
442             Node result = representers_[type](data, this);
443 
444             //Override tag if specified.
445             if(data.tag_ !is null){result.tag_ = data.tag_;}
446 
447             //Remember style if this was loaded before.
448             if(data.scalarStyle != ScalarStyle.Invalid)
449             {
450                 result.scalarStyle = data.scalarStyle;
451             }
452             if(data.collectionStyle != CollectionStyle.Invalid)
453             {
454                 result.collectionStyle = data.collectionStyle;
455             }
456             return result;
457         }
458 
459         //Represent a node, serializing with specified Serializer.
460         void represent(Range, CharType)(ref Serializer!(Range, CharType) serializer, ref Node node) @safe
461         {
462             auto data = representData(node);
463             serializer.serialize(data);
464         }
465 }
466 
467 
468 ///Represent a _null _node as a _null YAML value.
469 Node representNull(ref Node node, Representer representer) @safe
470 {
471     return representer.representScalar("tag:yaml.org,2002:null", "null");
472 }
473 
474 ///Represent a string _node as a string scalar.
475 Node representString(ref Node node, Representer representer) @safe
476 {
477     string value = node.as!string;
478     return value is null
479            ? representNull(node, representer)
480            : representer.representScalar("tag:yaml.org,2002:str", value);
481 }
482 
483 ///Represent a bytes _node as a binary scalar.
484 Node representBytes(ref Node node, Representer representer) @safe
485 {
486     const ubyte[] value = node.as!(ubyte[]);
487     if(value is null){return representNull(node, representer);}
488     return representer.representScalar("tag:yaml.org,2002:binary",
489                                        Base64.encode(value).idup,
490                                        ScalarStyle.Literal);
491 }
492 
493 ///Represent a bool _node as a bool scalar.
494 Node representBool(ref Node node, Representer representer) @safe
495 {
496     return representer.representScalar("tag:yaml.org,2002:bool",
497                                        node.as!bool ? "true" : "false");
498 }
499 
500 ///Represent a long _node as an integer scalar.
501 Node representLong(ref Node node, Representer representer) @safe
502 {
503     return representer.representScalar("tag:yaml.org,2002:int",
504                                        to!string(node.as!long));
505 }
506 
507 ///Represent a real _node as a floating point scalar.
508 Node representReal(ref Node node, Representer representer) @safe
509 {
510     real f = node.as!real;
511     string value = isNaN(f)                  ? ".nan":
512                    f == real.infinity        ? ".inf":
513                    f == -1.0 * real.infinity ? "-.inf":
514                    {auto a = appender!string();
515                     formattedWrite(a, "%12f", f);
516                     return a.data.strip();}();
517 
518     return representer.representScalar("tag:yaml.org,2002:float", value);
519 }
520 
521 ///Represent a SysTime _node as a timestamp.
522 Node representSysTime(ref Node node, Representer representer) @safe
523 {
524     return representer.representScalar("tag:yaml.org,2002:timestamp",
525                                        node.as!SysTime.toISOExtString());
526 }
527 
528 ///Represent a sequence _node as sequence/set.
529 Node representNodes(ref Node node, Representer representer) @safe
530 {
531     auto nodes = node.as!(Node[]);
532     if(node.tag_ == "tag:yaml.org,2002:set")
533     {
534         ///YAML sets are mapping with null values.
535         Node.Pair[] pairs;
536         pairs.length = nodes.length;
537         Node dummy;
538         foreach(idx, ref key; nodes)
539         {
540             pairs[idx] = Node.Pair(key, representNull(dummy, representer));
541         }
542         return representer.representMapping(node.tag_, pairs);
543     }
544     else
545     {
546         return representer.representSequence("tag:yaml.org,2002:seq", nodes);
547     }
548 }
549 
550 ///Represent a mapping _node as map/ordered map/pairs.
551 Node representPairs(ref Node node, Representer representer) @safe
552 {
553     auto pairs = node.as!(Node.Pair[]);
554 
555     bool hasDuplicates(Node.Pair[] pairs) @safe
556     {
557         //TODO this should be replaced by something with deterministic memory allocation.
558         auto keys = redBlackTree!Node();
559         foreach(ref pair; pairs)
560         {
561             if(pair.key in keys){return true;}
562             keys.insert(pair.key);
563         }
564         return false;
565     }
566 
567     Node[] mapToSequence(Node.Pair[] pairs) @safe
568     {
569         Node[] nodes;
570         nodes.length = pairs.length;
571         foreach(idx, ref pair; pairs)
572         {
573             nodes[idx] = representer.representMapping("tag:yaml.org,2002:map", [pair]);
574         }
575         return nodes;
576     }
577 
578     if(node.tag_ == "tag:yaml.org,2002:omap")
579     {
580         enforce(!hasDuplicates(pairs),
581                 new RepresenterException("Duplicate entry in an ordered map"));
582         return representer.representSequence(node.tag_, mapToSequence(pairs));
583     }
584     else if(node.tag_ == "tag:yaml.org,2002:pairs")
585     {
586         return representer.representSequence(node.tag_, mapToSequence(pairs));
587     }
588     else
589     {
590         enforce(!hasDuplicates(pairs),
591                 new RepresenterException("Duplicate entry in an unordered map"));
592         return representer.representMapping("tag:yaml.org,2002:map", pairs);
593     }
594 }
595 
596 //Unittests
597 //These should really all be encapsulated in unittests.
598 private:
599 
600 import dyaml.dumper;
601 
602 struct MyStruct
603 {
604     int x, y, z;
605 
606     int opCmp(ref const MyStruct s) const pure @safe nothrow
607     {
608         if(x != s.x){return x - s.x;}
609         if(y != s.y){return y - s.y;}
610         if(z != s.z){return z - s.z;}
611         return 0;
612     }
613 }
614 
615 Node representMyStruct(ref Node node, Representer representer) @safe
616 {
617     //The node is guaranteed to be MyStruct as we add representer for MyStruct.
618     auto value = node.as!MyStruct;
619     //Using custom scalar format, x:y:z.
620     auto scalar = format("%s:%s:%s", value.x, value.y, value.z);
621     //Representing as a scalar, with custom tag to specify this data type.
622     return representer.representScalar("!mystruct.tag", scalar);
623 }
624 
625 Node representMyStructSeq(ref Node node, Representer representer) @safe
626 {
627     auto value = node.as!MyStruct;
628     auto nodes = [Node(value.x), Node(value.y), Node(value.z)];
629     return representer.representSequence("!mystruct.tag", nodes);
630 }
631 
632 Node representMyStructMap(ref Node node, Representer representer) @safe
633 {
634     auto value = node.as!MyStruct;
635     auto pairs = [Node.Pair("x", value.x),
636                   Node.Pair("y", value.y),
637                   Node.Pair("z", value.z)];
638     return representer.representMapping("!mystruct.tag", pairs);
639 }
640 
641 class MyClass
642 {
643     int x, y, z;
644 
645     this(int x, int y, int z) pure @safe nothrow
646     {
647         this.x = x;
648         this.y = y;
649         this.z = z;
650     }
651 
652     override int opCmp(Object o) pure @safe nothrow
653     {
654         MyClass s = cast(MyClass)o;
655         if(s is null){return -1;}
656         if(x != s.x){return x - s.x;}
657         if(y != s.y){return y - s.y;}
658         if(z != s.z){return z - s.z;}
659         return 0;
660     }
661 
662     ///Useful for Node.as!string .
663     override string toString() @safe
664     {
665         return format("MyClass(%s, %s, %s)", x, y, z);
666     }
667 }
668 
669 //Same as representMyStruct.
670 Node representMyClass(ref Node node, Representer representer) @safe
671 {
672     //The node is guaranteed to be MyClass as we add representer for MyClass.
673     auto value = node.as!MyClass;
674     //Using custom scalar format, x:y:z.
675     auto scalar = format("%s:%s:%s", value.x, value.y, value.z);
676     //Representing as a scalar, with custom tag to specify this data type.
677     return representer.representScalar("!myclass.tag", scalar);
678 }
679 
680 @safe unittest
681 {
682     foreach(r; [&representMyStruct,
683                 &representMyStructSeq,
684                 &representMyStructMap])
685     {
686         auto dumper = dumper(new Appender!string);
687         auto representer = new Representer;
688         representer.addRepresenter!MyStruct(r);
689         dumper.representer = representer;
690         dumper.dump(Node(MyStruct(1,2,3)));
691     }
692 }
693 
694 @safe unittest
695 {
696     auto dumper = dumper(new Appender!string);
697     auto representer = new Representer;
698     representer.addRepresenter!MyClass(&representMyClass);
699     dumper.representer = representer;
700     dumper.dump(Node(new MyClass(1,2,3)));
701 }