dyaml.test.tokens source code

1 
2 //          Copyright Ferdinand Majerech 2011.
3 // Distributed under the Boost Software License, Version 1.0.
4 //    (See accompanying file LICENSE_1_0.txt or copy at
5 //          http://www.boost.org/LICENSE_1_0.txt)
6 
7 module dyaml.test.tokens;
8 
9 
10 version(unittest)
11 {
12 
13 import std.array;
14 import std.file;
15 
16 import dyaml.test.common;
17 import dyaml.reader;
18 import dyaml.scanner;
19 import dyaml.token;
20 
21 // Read and scan a YAML doc, returning the tokens.
22 const(Token)[] scanTestCommon(string filename) @safe
23 {
24     ubyte[] yamlData;
25     () @trusted { yamlData = cast(ubyte[])std.file.read(filename); }();
26     auto scanner = Scanner(new Reader(yamlData));
27     const(Token)[] result;
28     foreach (token; scanner)
29     {
30         result ~= token;
31     }
32     return result;
33 }
34 
35 /**
36  * Test tokens output by scanner.
37  *
38  * Params:  dataFilename   = File to scan.
39  *          tokensFilename = File containing expected tokens.
40  */
41 void testTokens(string dataFilename, string tokensFilename) @safe
42 {
43     //representations of YAML tokens in tokens file.
44     auto replace = [
45         TokenID.directive: "%",
46         TokenID.documentStart: "---",
47         TokenID.documentEnd: "...",
48         TokenID.alias_: "*",
49         TokenID.anchor: "&",
50         TokenID.tag: "!",
51         TokenID.scalar: "_",
52         TokenID.blockSequenceStart: "[[",
53         TokenID.blockMappingStart: "{{",
54         TokenID.blockEnd: "]}",
55         TokenID.flowSequenceStart: "[",
56         TokenID.flowSequenceEnd: "]",
57         TokenID.flowMappingStart: "{",
58         TokenID.flowMappingEnd: "}",
59         TokenID.blockEntry: ",",
60         TokenID.flowEntry: ",",
61         TokenID.key: "?",
62         TokenID.value: ":"
63     ];
64 
65     string[] tokens1;
66     string[] tokens2 = readText(tokensFilename).split();
67     scope(exit)
68     {
69         static if(verbose){writeln("tokens1: ", tokens1, "\ntokens2: ", tokens2);}
70     }
71 
72     foreach(token; scanTestCommon(dataFilename))
73     {
74         if(token.id != TokenID.streamStart && token.id != TokenID.streamEnd)
75         {
76             tokens1 ~= replace[token.id];
77         }
78     }
79 
80     assert(tokens1 == tokens2);
81 }
82 
83 /**
84  * Test scanner by scanning a file, expecting no errors.
85  *
86  * Params:  dataFilename      = File to scan.
87  *          canonicalFilename = Another file to scan, in canonical YAML format.
88  */
89 void testScanner(string dataFilename, string canonicalFilename) @safe
90 {
91     foreach(filename; [dataFilename, canonicalFilename])
92     {
93         string[] tokens;
94         scope(exit)
95         {
96             static if(verbose){writeln(tokens);}
97         }
98         foreach(ref token; scanTestCommon(filename))
99         {
100             tokens ~= to!string(token.id);
101         }
102     }
103 }
104 
105 @safe unittest
106 {
107     printProgress("D:YAML tokens unittest");
108     run("testTokens",  &testTokens, ["data", "tokens"]);
109     run("testScanner", &testScanner, ["data", "canonical"]);
110 }
111 
112 } // version(unittest)