aboutsummaryrefslogtreecommitdiffhomepage
path: root/src/sdlang/token.d
diff options
context:
space:
mode:
Diffstat (limited to 'src/sdlang/token.d')
-rw-r--r--src/sdlang/token.d550
1 files changed, 0 insertions, 550 deletions
diff --git a/src/sdlang/token.d b/src/sdlang/token.d
deleted file mode 100644
index 0a5b2fd..0000000
--- a/src/sdlang/token.d
+++ /dev/null
@@ -1,550 +0,0 @@
-// SDLang-D
-// Written in the D programming language.
-
-module sdlang.token;
-
-import std.array;
-import std.base64;
-import std.conv;
-import std.datetime;
-import std.meta;
-import std.range;
-import std.string;
-import std.traits;
-import std.typetuple;
-import std.variant;
-
-import sdlang.exception;
-import sdlang.symbol;
-import sdlang.util;
-
-/// DateTime doesn't support milliseconds, but SDLang's "Date Time" type does.
-/// So this is needed for any SDL "Date Time" that doesn't include a time zone.
-struct DateTimeFrac
-{
- DateTime dateTime;
- Duration fracSecs;
- deprecated("Use fracSecs instead.") {
- @property FracSec fracSec() const { return FracSec.from!"hnsecs"(fracSecs.total!"hnsecs"); }
- @property void fracSec(FracSec v) { fracSecs = v.hnsecs.hnsecs; }
- }
-}
-
-/++
-If a "Date Time" literal in the SDL file has a time zone that's not found in
-your system, you get one of these instead of a SysTime. (Because it's
-impossible to indicate "unknown time zone" with `std.datetime.TimeZone`.)
-
-The difference between this and `DateTimeFrac` is that `DateTimeFrac`
-indicates that no time zone was specified in the SDL at all, whereas
-`DateTimeFracUnknownZone` indicates that a time zone was specified but
-data for it could not be found on your system.
-+/
-struct DateTimeFracUnknownZone
-{
- DateTime dateTime;
- Duration fracSecs;
- deprecated("Use fracSecs instead.") {
- @property FracSec fracSec() const { return FracSec.from!"hnsecs"(fracSecs.total!"hnsecs"); }
- @property void fracSec(FracSec v) { fracSecs = v.hnsecs.hnsecs; }
- }
- string timeZone;
-
- bool opEquals(const DateTimeFracUnknownZone b) const
- {
- return opEquals(b);
- }
- bool opEquals(ref const DateTimeFracUnknownZone b) const
- {
- return
- this.dateTime == b.dateTime &&
- this.fracSecs == b.fracSecs &&
- this.timeZone == b.timeZone;
- }
-}
-
-/++
-SDLang's datatypes map to D's datatypes as described below.
-Most are straightforward, but take special note of the date/time-related types.
-
----------------------------------------------------------------
-Boolean: bool
-Null: typeof(null)
-Unicode Character: dchar
-Double-Quote Unicode String: string
-Raw Backtick Unicode String: string
-Integer (32 bits signed): int
-Long Integer (64 bits signed): long
-Float (32 bits signed): float
-Double Float (64 bits signed): double
-Decimal (128+ bits signed): real
-Binary (standard Base64): ubyte[]
-Time Span: Duration
-
-Date (with no time at all): Date
-Date Time (no timezone): DateTimeFrac
-Date Time (with a known timezone): SysTime
-Date Time (with an unknown timezone): DateTimeFracUnknownZone
----------------------------------------------------------------
-+/
-alias ValueTypes = TypeTuple!(
- bool,
- string, dchar,
- int, long,
- float, double, real,
- Date, DateTimeFrac, SysTime, DateTimeFracUnknownZone, Duration,
- ubyte[],
- typeof(null),
-);
-
-alias Value = Algebraic!( ValueTypes ); ///ditto
-enum isValueType(T) = staticIndexOf!(T, ValueTypes) != -1;
-
-enum isSink(T) =
- isOutputRange!T &&
- is(ElementType!(T)[] == string);
-
-string toSDLString(T)(T value) if(is(T==Value) || isValueType!T)
-{
- Appender!string sink;
- toSDLString(value, sink);
- return sink.data;
-}
-
-/// Throws SDLangException if value is infinity, -infinity or NaN, because
-/// those are not currently supported by the SDLang spec.
-void toSDLString(Sink)(Value value, ref Sink sink) if(isOutputRange!(Sink,char))
-{
- foreach(T; ValueTypes)
- {
- if(value.type == typeid(T))
- {
- toSDLString( value.get!T(), sink );
- return;
- }
- }
-
- throw new Exception("Internal SDLang-D error: Unhandled type of Value. Contains: "~value.toString());
-}
-
-@("toSDLString on infinity and NaN")
-unittest
-{
- import std.exception;
-
- auto floatInf = float.infinity;
- auto floatNegInf = -float.infinity;
- auto floatNaN = float.nan;
-
- auto doubleInf = double.infinity;
- auto doubleNegInf = -double.infinity;
- auto doubleNaN = double.nan;
-
- auto realInf = real.infinity;
- auto realNegInf = -real.infinity;
- auto realNaN = real.nan;
-
- assertNotThrown( toSDLString(0.0F) );
- assertNotThrown( toSDLString(0.0) );
- assertNotThrown( toSDLString(0.0L) );
-
- assertThrown!ValidationException( toSDLString(floatInf) );
- assertThrown!ValidationException( toSDLString(floatNegInf) );
- assertThrown!ValidationException( toSDLString(floatNaN) );
-
- assertThrown!ValidationException( toSDLString(doubleInf) );
- assertThrown!ValidationException( toSDLString(doubleNegInf) );
- assertThrown!ValidationException( toSDLString(doubleNaN) );
-
- assertThrown!ValidationException( toSDLString(realInf) );
- assertThrown!ValidationException( toSDLString(realNegInf) );
- assertThrown!ValidationException( toSDLString(realNaN) );
-
- assertThrown!ValidationException( toSDLString(Value(floatInf)) );
- assertThrown!ValidationException( toSDLString(Value(floatNegInf)) );
- assertThrown!ValidationException( toSDLString(Value(floatNaN)) );
-
- assertThrown!ValidationException( toSDLString(Value(doubleInf)) );
- assertThrown!ValidationException( toSDLString(Value(doubleNegInf)) );
- assertThrown!ValidationException( toSDLString(Value(doubleNaN)) );
-
- assertThrown!ValidationException( toSDLString(Value(realInf)) );
- assertThrown!ValidationException( toSDLString(Value(realNegInf)) );
- assertThrown!ValidationException( toSDLString(Value(realNaN)) );
-}
-
-void toSDLString(Sink)(typeof(null) value, ref Sink sink) if(isOutputRange!(Sink,char))
-{
- sink.put("null");
-}
-
-void toSDLString(Sink)(bool value, ref Sink sink) if(isOutputRange!(Sink,char))
-{
- sink.put(value? "true" : "false");
-}
-
-//TODO: Figure out how to properly handle strings/chars containing lineSep or paraSep
-void toSDLString(Sink)(string value, ref Sink sink) if(isOutputRange!(Sink,char))
-{
- sink.put('"');
-
- // This loop is UTF-safe
- foreach(char ch; value)
- {
- if (ch == '\n') sink.put(`\n`);
- else if(ch == '\r') sink.put(`\r`);
- else if(ch == '\t') sink.put(`\t`);
- else if(ch == '\"') sink.put(`\"`);
- else if(ch == '\\') sink.put(`\\`);
- else
- sink.put(ch);
- }
-
- sink.put('"');
-}
-
-void toSDLString(Sink)(dchar value, ref Sink sink) if(isOutputRange!(Sink,char))
-{
- sink.put('\'');
-
- if (value == '\n') sink.put(`\n`);
- else if(value == '\r') sink.put(`\r`);
- else if(value == '\t') sink.put(`\t`);
- else if(value == '\'') sink.put(`\'`);
- else if(value == '\\') sink.put(`\\`);
- else
- sink.put(value);
-
- sink.put('\'');
-}
-
-void toSDLString(Sink)(int value, ref Sink sink) if(isOutputRange!(Sink,char))
-{
- sink.put( "%s".format(value) );
-}
-
-void toSDLString(Sink)(long value, ref Sink sink) if(isOutputRange!(Sink,char))
-{
- sink.put( "%sL".format(value) );
-}
-
-private void checkUnsupportedFloatingPoint(T)(T value) if(isFloatingPoint!T)
-{
- import std.exception;
- import std.math;
-
- enforce!ValidationException(
- !isInfinity(value),
- "SDLang does not currently support infinity for floating-point types"
- );
-
- enforce!ValidationException(
- !isNaN(value),
- "SDLang does not currently support NaN for floating-point types"
- );
-}
-
-void toSDLString(Sink)(float value, ref Sink sink) if(isOutputRange!(Sink,char))
-{
- checkUnsupportedFloatingPoint(value);
- sink.put( "%.10sF".format(value) );
-}
-
-void toSDLString(Sink)(double value, ref Sink sink) if(isOutputRange!(Sink,char))
-{
- checkUnsupportedFloatingPoint(value);
- sink.put( "%.30sD".format(value) );
-}
-
-void toSDLString(Sink)(real value, ref Sink sink) if(isOutputRange!(Sink,char))
-{
- checkUnsupportedFloatingPoint(value);
- sink.put( "%.30sBD".format(value) );
-}
-
-void toSDLString(Sink)(Date value, ref Sink sink) if(isOutputRange!(Sink,char))
-{
- sink.put(to!string(value.year));
- sink.put('/');
- sink.put(to!string(cast(int)value.month));
- sink.put('/');
- sink.put(to!string(value.day));
-}
-
-void toSDLString(Sink)(DateTimeFrac value, ref Sink sink) if(isOutputRange!(Sink,char))
-{
- toSDLString(value.dateTime.date, sink);
- sink.put(' ');
- sink.put("%.2s".format(value.dateTime.hour));
- sink.put(':');
- sink.put("%.2s".format(value.dateTime.minute));
-
- if(value.dateTime.second != 0)
- {
- sink.put(':');
- sink.put("%.2s".format(value.dateTime.second));
- }
-
- if(value.fracSecs != 0.msecs)
- {
- sink.put('.');
- sink.put("%.3s".format(value.fracSecs.total!"msecs"));
- }
-}
-
-void toSDLString(Sink)(SysTime value, ref Sink sink) if(isOutputRange!(Sink,char))
-{
- auto dateTimeFrac = DateTimeFrac(cast(DateTime)value, value.fracSecs);
- toSDLString(dateTimeFrac, sink);
-
- sink.put("-");
-
- auto tzString = value.timezone.name;
-
- // If name didn't exist, try abbreviation.
- // Note that according to std.datetime docs, on Windows the
- // stdName/dstName may not be properly abbreviated.
- version(Windows) {} else
- if(tzString == "")
- {
- auto tz = value.timezone;
- auto stdTime = value.stdTime;
-
- if(tz.hasDST())
- tzString = tz.dstInEffect(stdTime)? tz.dstName : tz.stdName;
- else
- tzString = tz.stdName;
- }
-
- if(tzString == "")
- {
- auto offset = value.timezone.utcOffsetAt(value.stdTime);
- sink.put("GMT");
-
- if(offset < seconds(0))
- {
- sink.put("-");
- offset = -offset;
- }
- else
- sink.put("+");
-
- sink.put("%.2s".format(offset.split.hours));
- sink.put(":");
- sink.put("%.2s".format(offset.split.minutes));
- }
- else
- sink.put(tzString);
-}
-
-void toSDLString(Sink)(DateTimeFracUnknownZone value, ref Sink sink) if(isOutputRange!(Sink,char))
-{
- auto dateTimeFrac = DateTimeFrac(value.dateTime, value.fracSecs);
- toSDLString(dateTimeFrac, sink);
-
- sink.put("-");
- sink.put(value.timeZone);
-}
-
-void toSDLString(Sink)(Duration value, ref Sink sink) if(isOutputRange!(Sink,char))
-{
- if(value < seconds(0))
- {
- sink.put("-");
- value = -value;
- }
-
- auto days = value.total!"days"();
- if(days != 0)
- {
- sink.put("%s".format(days));
- sink.put("d:");
- }
-
- sink.put("%.2s".format(value.split.hours));
- sink.put(':');
- sink.put("%.2s".format(value.split.minutes));
- sink.put(':');
- sink.put("%.2s".format(value.split.seconds));
-
- if(value.split.msecs != 0)
- {
- sink.put('.');
- sink.put("%.3s".format(value.split.msecs));
- }
-}
-
-void toSDLString(Sink)(ubyte[] value, ref Sink sink) if(isOutputRange!(Sink,char))
-{
- sink.put('[');
- sink.put( Base64.encode(value) );
- sink.put(']');
-}
-
-/// This only represents terminals. Nonterminals aren't
-/// constructed since the AST is directly built during parsing.
-struct Token
-{
- Symbol symbol = sdlang.symbol.symbol!"Error"; /// The "type" of this token
- Location location;
- Value value; /// Only valid when `symbol` is `symbol!"Value"`, otherwise null
- string data; /// Original text from source
-
- @disable this();
- this(Symbol symbol, Location location, Value value=Value(null), string data=null)
- {
- this.symbol = symbol;
- this.location = location;
- this.value = value;
- this.data = data;
- }
-
- /// Tokens with differing symbols are always unequal.
- /// Tokens with differing values are always unequal.
- /// Tokens with differing Value types are always unequal.
- /// Member `location` is always ignored for comparison.
- /// Member `data` is ignored for comparison *EXCEPT* when the symbol is Ident.
- bool opEquals(Token b)
- {
- return opEquals(b);
- }
- bool opEquals(ref Token b) ///ditto
- {
- if(
- this.symbol != b.symbol ||
- this.value.type != b.value.type ||
- this.value != b.value
- )
- return false;
-
- if(this.symbol == .symbol!"Ident")
- return this.data == b.data;
-
- return true;
- }
-
- bool matches(string symbolName)()
- {
- return this.symbol == .symbol!symbolName;
- }
-}
-
-@("sdlang token")
-unittest
-{
- auto loc = Location("", 0, 0, 0);
- auto loc2 = Location("a", 1, 1, 1);
-
- assert(Token(symbol!"EOL",loc) == Token(symbol!"EOL",loc ));
- assert(Token(symbol!"EOL",loc) == Token(symbol!"EOL",loc2));
- assert(Token(symbol!":", loc) == Token(symbol!":", loc ));
- assert(Token(symbol!"EOL",loc) != Token(symbol!":", loc ));
- assert(Token(symbol!"EOL",loc,Value(null),"\n") == Token(symbol!"EOL",loc,Value(null),"\n"));
-
- assert(Token(symbol!"EOL",loc,Value(null),"\n") == Token(symbol!"EOL",loc,Value(null),";" ));
- assert(Token(symbol!"EOL",loc,Value(null),"A" ) == Token(symbol!"EOL",loc,Value(null),"B" ));
- assert(Token(symbol!":", loc,Value(null),"A" ) == Token(symbol!":", loc,Value(null),"BB"));
- assert(Token(symbol!"EOL",loc,Value(null),"A" ) != Token(symbol!":", loc,Value(null),"A" ));
-
- assert(Token(symbol!"Ident",loc,Value(null),"foo") == Token(symbol!"Ident",loc,Value(null),"foo"));
- assert(Token(symbol!"Ident",loc,Value(null),"foo") != Token(symbol!"Ident",loc,Value(null),"BAR"));
-
- assert(Token(symbol!"Value",loc,Value(null),"foo") == Token(symbol!"Value",loc, Value(null),"foo"));
- assert(Token(symbol!"Value",loc,Value(null),"foo") == Token(symbol!"Value",loc2,Value(null),"foo"));
- assert(Token(symbol!"Value",loc,Value(null),"foo") == Token(symbol!"Value",loc, Value(null),"BAR"));
- assert(Token(symbol!"Value",loc,Value( 7),"foo") == Token(symbol!"Value",loc, Value( 7),"BAR"));
- assert(Token(symbol!"Value",loc,Value( 7),"foo") != Token(symbol!"Value",loc, Value( "A"),"foo"));
- assert(Token(symbol!"Value",loc,Value( 7),"foo") != Token(symbol!"Value",loc, Value( 2),"foo"));
- assert(Token(symbol!"Value",loc,Value(cast(int)7)) != Token(symbol!"Value",loc, Value(cast(long)7)));
- assert(Token(symbol!"Value",loc,Value(cast(float)1.2)) != Token(symbol!"Value",loc, Value(cast(double)1.2)));
-}
-
-@("sdlang Value.toSDLString()")
-unittest
-{
- // Bool and null
- assert(Value(null ).toSDLString() == "null");
- assert(Value(true ).toSDLString() == "true");
- assert(Value(false).toSDLString() == "false");
-
- // Base64 Binary
- assert(Value(cast(ubyte[])"hello world".dup).toSDLString() == "[aGVsbG8gd29ybGQ=]");
-
- // Integer
- assert(Value(cast( int) 7).toSDLString() == "7");
- assert(Value(cast( int)-7).toSDLString() == "-7");
- assert(Value(cast( int) 0).toSDLString() == "0");
-
- assert(Value(cast(long) 7).toSDLString() == "7L");
- assert(Value(cast(long)-7).toSDLString() == "-7L");
- assert(Value(cast(long) 0).toSDLString() == "0L");
-
- // Floating point
- assert(Value(cast(float) 1.5).toSDLString() == "1.5F");
- assert(Value(cast(float)-1.5).toSDLString() == "-1.5F");
- assert(Value(cast(float) 0).toSDLString() == "0F");
-
- assert(Value(cast(double) 1.5).toSDLString() == "1.5D");
- assert(Value(cast(double)-1.5).toSDLString() == "-1.5D");
- assert(Value(cast(double) 0).toSDLString() == "0D");
-
- assert(Value(cast(real) 1.5).toSDLString() == "1.5BD");
- assert(Value(cast(real)-1.5).toSDLString() == "-1.5BD");
- assert(Value(cast(real) 0).toSDLString() == "0BD");
-
- // String
- assert(Value("hello" ).toSDLString() == `"hello"`);
- assert(Value(" hello ").toSDLString() == `" hello "`);
- assert(Value("" ).toSDLString() == `""`);
- assert(Value("hello \r\n\t\"\\ world").toSDLString() == `"hello \r\n\t\"\\ world"`);
- assert(Value("日本語").toSDLString() == `"日本語"`);
-
- // Chars
- assert(Value(cast(dchar) 'A').toSDLString() == `'A'`);
- assert(Value(cast(dchar)'\r').toSDLString() == `'\r'`);
- assert(Value(cast(dchar)'\n').toSDLString() == `'\n'`);
- assert(Value(cast(dchar)'\t').toSDLString() == `'\t'`);
- assert(Value(cast(dchar)'\'').toSDLString() == `'\''`);
- assert(Value(cast(dchar)'\\').toSDLString() == `'\\'`);
- assert(Value(cast(dchar) '月').toSDLString() == `'月'`);
-
- // Date
- assert(Value(Date( 2004,10,31)).toSDLString() == "2004/10/31");
- assert(Value(Date(-2004,10,31)).toSDLString() == "-2004/10/31");
-
- // DateTimeFrac w/o Frac
- assert(Value(DateTimeFrac(DateTime(2004,10,31, 14,30,15))).toSDLString() == "2004/10/31 14:30:15");
- assert(Value(DateTimeFrac(DateTime(2004,10,31, 1, 2, 3))).toSDLString() == "2004/10/31 01:02:03");
- assert(Value(DateTimeFrac(DateTime(-2004,10,31, 14,30,15))).toSDLString() == "-2004/10/31 14:30:15");
-
- // DateTimeFrac w/ Frac
- assert(Value(DateTimeFrac(DateTime(2004,10,31, 14,30,15), 123.msecs)).toSDLString() == "2004/10/31 14:30:15.123");
- assert(Value(DateTimeFrac(DateTime(2004,10,31, 14,30,15), 120.msecs)).toSDLString() == "2004/10/31 14:30:15.120");
- assert(Value(DateTimeFrac(DateTime(2004,10,31, 14,30,15), 100.msecs)).toSDLString() == "2004/10/31 14:30:15.100");
- assert(Value(DateTimeFrac(DateTime(2004,10,31, 14,30,15), 12.msecs)).toSDLString() == "2004/10/31 14:30:15.012");
- assert(Value(DateTimeFrac(DateTime(2004,10,31, 14,30,15), 1.msecs)).toSDLString() == "2004/10/31 14:30:15.001");
- assert(Value(DateTimeFrac(DateTime(-2004,10,31, 14,30,15), 123.msecs)).toSDLString() == "-2004/10/31 14:30:15.123");
-
- // DateTimeFracUnknownZone
- assert(Value(DateTimeFracUnknownZone(DateTime(2004,10,31, 14,30,15), 123.msecs, "Foo/Bar")).toSDLString() == "2004/10/31 14:30:15.123-Foo/Bar");
-
- // SysTime
- assert(Value(SysTime(DateTime(2004,10,31, 14,30,15), new immutable SimpleTimeZone( hours(0) ))).toSDLString() == "2004/10/31 14:30:15-GMT+00:00");
- assert(Value(SysTime(DateTime(2004,10,31, 1, 2, 3), new immutable SimpleTimeZone( hours(0) ))).toSDLString() == "2004/10/31 01:02:03-GMT+00:00");
- assert(Value(SysTime(DateTime(2004,10,31, 14,30,15), new immutable SimpleTimeZone( hours(2)+minutes(10) ))).toSDLString() == "2004/10/31 14:30:15-GMT+02:10");
- assert(Value(SysTime(DateTime(2004,10,31, 14,30,15), new immutable SimpleTimeZone(-hours(5)-minutes(30) ))).toSDLString() == "2004/10/31 14:30:15-GMT-05:30");
- assert(Value(SysTime(DateTime(2004,10,31, 14,30,15), new immutable SimpleTimeZone( hours(2)+minutes( 3) ))).toSDLString() == "2004/10/31 14:30:15-GMT+02:03");
- assert(Value(SysTime(DateTime(2004,10,31, 14,30,15), 123.msecs, new immutable SimpleTimeZone( hours(0) ))).toSDLString() == "2004/10/31 14:30:15.123-GMT+00:00");
-
- // Duration
- assert( "12:14:42" == Value( days( 0)+hours(12)+minutes(14)+seconds(42)+msecs( 0)).toSDLString());
- assert("-12:14:42" == Value(-days( 0)-hours(12)-minutes(14)-seconds(42)-msecs( 0)).toSDLString());
- assert( "00:09:12" == Value( days( 0)+hours( 0)+minutes( 9)+seconds(12)+msecs( 0)).toSDLString());
- assert( "00:00:01.023" == Value( days( 0)+hours( 0)+minutes( 0)+seconds( 1)+msecs( 23)).toSDLString());
- assert( "23d:05:21:23.532" == Value( days(23)+hours( 5)+minutes(21)+seconds(23)+msecs(532)).toSDLString());
- assert( "23d:05:21:23.530" == Value( days(23)+hours( 5)+minutes(21)+seconds(23)+msecs(530)).toSDLString());
- assert( "23d:05:21:23.500" == Value( days(23)+hours( 5)+minutes(21)+seconds(23)+msecs(500)).toSDLString());
- assert("-23d:05:21:23.532" == Value(-days(23)-hours( 5)-minutes(21)-seconds(23)-msecs(532)).toSDLString());
- assert("-23d:05:21:23.500" == Value(-days(23)-hours( 5)-minutes(21)-seconds(23)-msecs(500)).toSDLString());
- assert( "23d:05:21:23" == Value( days(23)+hours( 5)+minutes(21)+seconds(23)+msecs( 0)).toSDLString());
-}
generated by cgit v1.2.3 (git 2.25.1) at 2024-07-13 04:08:09 +0000