source/factorcss/Tokeniser.x

--
-- This file is part of FactorCSS
-- Licensed under the MIT License,
--                http://www.opensource.org/licenses/mit-license
-- Copyright 2004 James Bursa <james@semichrome.net>
--

-- Tokeniser for CSS 2.1
-- See CSS 2.1 G.2

{
module Tokeniser where
}

%wrapper "basic"

$h              = [0-9a-f]
$nonascii       = [\x80-\xff]
@unicode        = \\$h{1,6}(\r\n|[\ \t\r\n\f])?
@escape         = @unicode|\\[\x20-\x7e\x80-\xff]
@nmstart        = [_a-z]|$nonascii|@escape
@nmchar         = [\-_a-zA-Z0-9]|$nonascii|@escape
@nl             = \n|\r\n|\r|\f
@string1        = \"([\t\ !#\x24\x25&\x28-\x7e]|\\@nl|\'|$nonascii|@escape|\.)*\"
@string2        = \'([\t\ !#\x24\x25&\x28-\x7e]|\\@nl|\"|$nonascii|@escape|\.)*\'
@ident          = @nmstart@nmchar*
@name           = @nmchar+
@num            = [\-\053]?[0-9]+|[0-9]*"."[0-9]+
@string         = @string1|@string2
@url            = ([!#\x24\x25&\x2a-\x7e]|$nonascii|@escape)*
$s              = [\ \t\r\n\f]
@w              = $s*
@range          = \?{1,6}|$h(\?{0,5}|$h(\?{0,4}|$h(\?{0,3}|$h(\?{0,2}|$h(\??|$h)))))
@comment        = \/\*[^\x2a]*\*+([^\/\x2a][^\x2a]*\*+)*\/

tokens :-

$s+                     ;
@comment                ;

"<!--"                  ;
"-->"                   ;
"~="                    { \s -> INCLUDES }
"|="                    { \s -> DASHMATCH }

@w"{"                   { \s -> LBRACE }
@w"+"                   { \s -> PLUS }
@w">"                   { \s -> GREATER }
@w","                   { \s -> COMMA }

@string                 { \s -> STRING s }

@ident                  { \s -> IDENT s }

"#"@name                { \s -> HASH s }

"@import"               { \s -> IMPORT_SYM }
"@page"                 { \s -> PAGE_SYM }
"@media"                { \s -> MEDIA_SYM }
"@charset"              { \s -> CHARSET_SYM }

"!"@w"important"        { \s -> IMPORTANT_SYM }

@num em                 { \s -> EMS (read (take (length s - 2) s)) }
@num ex                 { \s -> EXS (read (take (length s - 2) s)) }
@num px                 { \s -> LENGTH s }
@num cm                 { \s -> LENGTH s }
@num mm                 { \s -> LENGTH s }
@num in                 { \s -> LENGTH s }
@num pt                 { \s -> LENGTH s }
@num pc                 { \s -> LENGTH s }
@num deg                { \s -> ANGLE s }
@num rad                { \s -> ANGLE s }
@num grad               { \s -> ANGLE s }
@num ms                 { \s -> TIME s }
@num s                  { \s -> TIME s }
@num Hz                 { \s -> FREQ s }
@num kHz                { \s -> FREQ s }
@num @ident             { \s -> DIMEN s }
@num "%"                { \s -> PERCENTAGE (read (take (length s - 1) s)) }
@num                    { \s -> NUMBER (read s) }

"url("@w@string@w")"    { \s -> URI s }
"url("@w@url@w")"       { \s -> URI s }
@ident"\050"            { \s -> FUNCTION s }

";"                     { \s -> SEMI }
"}"                     { \s -> RBRACE }
":"                     { \s -> COLON }
"/"                     { \s -> SLASH }
"-"                     { \s -> MINUS }
"."                     { \s -> DOT }
"*"                     { \s -> ASTERISK }
"["                     { \s -> LBRAC }
"]"                     { \s -> RBRAC }
"="                     { \s -> EQUALS }
")"                     { \s -> RPAREN }
.                       { \s -> DELIM (head s) }


{
data Token =
        INCLUDES        |
        DASHMATCH       |
        LBRACE          |
        PLUS            |
        GREATER         |
        COMMA           |
        STRING String   |
        IDENT String    |
        HASH String     |
        IMPORT_SYM      |
        PAGE_SYM        |
        MEDIA_SYM       |
        CHARSET_SYM     |
        IMPORTANT_SYM   |
        EMS Float       |
        EXS Float       |
        LENGTH String   |
        ANGLE String    |
        TIME String     |
        FREQ String     |
        DIMEN String    |
        PERCENTAGE Float |
        NUMBER Float    |
        URI String      |
        FUNCTION String |
        SEMI            |
        RBRACE          |
        COLON           |
        SLASH           |
        MINUS           |
        DOT             |
        ASTERISK        |
        LBRAC           |
        RBRAC           |
        EQUALS          |
        RPAREN          |
        DELIM Char
        deriving (Eq, Show)
}
1	james	63	--
2			-- This file is part of FactorCSS
3			-- Licensed under the MIT License,
4			-- http://www.opensource.org/licenses/mit-license
5			-- Copyright 2004 James Bursa <james@semichrome.net>
6			--
7
8	james	61	-- Tokeniser for CSS 2.1
9			-- See CSS 2.1 G.2
10
11			{
12			module Tokeniser where
13			}
14
15			%wrapper "basic"
16
17			$h = [0-9a-f]
18			$nonascii = [\x80-\xff]
19			@unicode = \\$h{1,6}(\r\n\|[\ \t\r\n\f])?
20			@escape = @unicode\|\\[\x20-\x7e\x80-\xff]
21			@nmstart = [_a-z]\|$nonascii\|@escape
22			@nmchar = [\-_a-zA-Z0-9]\|$nonascii\|@escape
23			@nl = \n\|\r\n\|\r\|\f
24			@string1 = \"([\t\ !#\x24\x25&\x28-\x7e]\|\\@nl\|\'\|$nonascii\|@escape\|\.)*\"
25			@string2 = \'([\t\ !#\x24\x25&\x28-\x7e]\|\\@nl\|\"\|$nonascii\|@escape\|\.)*\'
26			@ident = @nmstart@nmchar*
27			@name = @nmchar+
28			@num = [\-\053]?[0-9]+\|[0-9]*"."[0-9]+
29			@string = @string1\|@string2
30			@url = ([!#\x24\x25&\x2a-\x7e]\|$nonascii\|@escape)*
31			$s = [\ \t\r\n\f]
32			@w = $s*
33			@range = \?{1,6}\|$h(\?{0,5}\|$h(\?{0,4}\|$h(\?{0,3}\|$h(\?{0,2}\|$h(\??\|$h)))))
34			@comment = \/\[^\x2a]\+([^\/\x2a][^\x2a]\+)\/
35
36			tokens :-
37
38			$s+ ;
39			@comment ;
40
41			"<!--" ;
42			"-->" ;
43			"~=" { \s -> INCLUDES }
44			"\|=" { \s -> DASHMATCH }
45
46			@w"{" { \s -> LBRACE }
47			@w"+" { \s -> PLUS }
48			@w">" { \s -> GREATER }
49			@w"," { \s -> COMMA }
50
51			@string { \s -> STRING s }
52
53			@ident { \s -> IDENT s }
54
55			"#"@name { \s -> HASH s }
56
57			"@import" { \s -> IMPORT_SYM }
58			"@page" { \s -> PAGE_SYM }
59			"@media" { \s -> MEDIA_SYM }
60			"@charset" { \s -> CHARSET_SYM }
61
62			"!"@w"important" { \s -> IMPORTANT_SYM }
63
64			@num em { \s -> EMS (read (take (length s - 2) s)) }
65			@num ex { \s -> EXS (read (take (length s - 2) s)) }
66			@num px { \s -> LENGTH s }
67			@num cm { \s -> LENGTH s }
68			@num mm { \s -> LENGTH s }
69			@num in { \s -> LENGTH s }
70			@num pt { \s -> LENGTH s }
71			@num pc { \s -> LENGTH s }
72			@num deg { \s -> ANGLE s }
73			@num rad { \s -> ANGLE s }
74			@num grad { \s -> ANGLE s }
75			@num ms { \s -> TIME s }
76			@num s { \s -> TIME s }
77			@num Hz { \s -> FREQ s }
78			@num kHz { \s -> FREQ s }
79			@num @ident { \s -> DIMEN s }
80			@num "%" { \s -> PERCENTAGE (read (take (length s - 1) s)) }
81			@num { \s -> NUMBER (read s) }
82
83			"url("@w@string@w")" { \s -> URI s }
84			"url("@w@url@w")" { \s -> URI s }
85			@ident"\050" { \s -> FUNCTION s }
86
87			";" { \s -> SEMI }
88			"}" { \s -> RBRACE }
89			":" { \s -> COLON }
90			"/" { \s -> SLASH }
91			"-" { \s -> MINUS }
92			"." { \s -> DOT }
93			"*" { \s -> ASTERISK }
94			"[" { \s -> LBRAC }
95			"]" { \s -> RBRAC }
96			"=" { \s -> EQUALS }
97			")" { \s -> RPAREN }
98			. { \s -> DELIM (head s) }
99
100
101			{
102			data Token =
103			INCLUDES \|
104			DASHMATCH \|
105			LBRACE \|
106			PLUS \|
107			GREATER \|
108			COMMA \|
109			STRING String \|
110			IDENT String \|
111			HASH String \|
112			IMPORT_SYM \|
113			PAGE_SYM \|
114			MEDIA_SYM \|
115			CHARSET_SYM \|
116			IMPORTANT_SYM \|
117			EMS Float \|
118			EXS Float \|
119			LENGTH String \|
120			ANGLE String \|
121			TIME String \|
122			FREQ String \|
123			DIMEN String \|
124			PERCENTAGE Float \|
125			NUMBER Float \|
126			URI String \|
127			FUNCTION String \|
128			SEMI \|
129			RBRACE \|
130			COLON \|
131			SLASH \|
132			MINUS \|
133			DOT \|
134			ASTERISK \|
135			LBRAC \|
136			RBRAC \|
137			EQUALS \|
138			RPAREN \|
139			DELIM Char
140			deriving (Eq, Show)
141			}