source/factorcss/Tokeniser.x

--
-- This file is part of FactorCSS
-- Licensed under the MIT License,
--                http://www.opensource.org/licenses/mit-license
-- Copyright 2004 James Bursa <james@semichrome.net>
--

-- Tokeniser for CSS 2.1
-- See CSS 2.1 G.2

{
module Tokeniser where
}

%wrapper "posn"

$h              = [0-9a-f]
$nonascii       = [\x80-\xff]
@unicode        = \\$h{1,6}(\r\n|[\ \t\r\n\f])?
@escape         = @unicode|\\[\x20-\x7e\x80-\xff]
@nmstart        = [_a-zA-Z]|$nonascii|@escape
@nmchar         = [\-_a-zA-Z0-9]|$nonascii|@escape
@nl             = \n|\r\n|\r|\f
@string1        = \"([\t\ \!\#\x24\x25&\x28-\x7e]|\\@nl|\'|$nonascii|@escape|\.)*\"
@string2        = \'([\t\ \!\#\x24\x25&\x28-\x7e]|\\@nl|\"|$nonascii|@escape|\.)*\'
@ident          = @nmstart@nmchar*
@name           = @nmchar+
@num            = [\-\053]?[0-9]+|[0-9]*"."[0-9]+
@string         = @string1|@string2
@url            = ([\!\#\x24\x25&\x2a-\x7e]|$nonascii|@escape)*
$s              = [\ \t\r\n\f]
@w              = $s*
@range          = \?{1,6}|$h(\?{0,5}|$h(\?{0,4}|$h(\?{0,3}|$h(\?{0,2}|$h(\??|$h)))))
$all            = \x00-\xff
@comment        = \/\*$all#\x2a*\*+($all#[\/\x2a]$all#\x2a*\*+)*\/

tokens :-

$s+                     { \p s -> (S, p) }
@comment                ;

"<!--"                  ;
"-->"                   ;
"~="                    { \p s -> (INCLUDES, p) }
"|="                    { \p s -> (DASHMATCH, p) }

@w"{"                   { \p s -> (LBRACE, p) }
@w"}"                   { \p s -> (RBRACE, p) }
@w"+"                   { \p s -> (PLUS, p) }
@w">"                   { \p s -> (GREATER, p) }
@w","                   { \p s -> (COMMA, p) }

@string                 { \p s -> (STRING s, p) }

@ident                  { \p s -> (IDENT s, p) }

"#"@name                { \p s -> (HASH s, p) }

"@import"               { \p s -> (IMPORT_SYM, p) }
"@page"                 { \p s -> (PAGE_SYM, p) }
"@media"                { \p s -> (MEDIA_SYM, p) }
"@charset"              { \p s -> (CHARSET_SYM, p) }

"!"@w"important"        { \p s -> (IMPORTANT_SYM, p) }

@num em                 { \p s -> (EMS (read ('0':(take (length s - 2) s))), p) }
@num ex                 { \p s -> (EXS (read ('0':(take (length s - 2) s))), p) }
@num px                 { \p s -> (LENGTH s, p) }
@num cm                 { \p s -> (LENGTH s, p) }
@num mm                 { \p s -> (LENGTH s, p) }
@num in                 { \p s -> (LENGTH s, p) }
@num pt                 { \p s -> (LENGTH s, p) }
@num pc                 { \p s -> (LENGTH s, p) }
@num deg                { \p s -> (ANGLE s, p) }
@num rad                { \p s -> (ANGLE s, p) }
@num grad               { \p s -> (ANGLE s, p) }
@num ms                 { \p s -> (TIME s, p) }
@num s                  { \p s -> (TIME s, p) }
@num Hz                 { \p s -> (FREQ s, p) }
@num kHz                { \p s -> (FREQ s, p) }
@num @ident             { \p s -> (DIMEN s, p) }
@num "%"                { \p s -> (PERCENTAGE (read ('0':(take (length s - 1) s))), p) }
@num                    { \p s -> (NUMBER (read ('0':s)), p) }

"url("@w@string@w")"    { \p s -> (URI s, p) }
"url("@w@url@w")"       { \p s -> (URI s, p) }
@ident"\050"            { \p s -> (FUNCTION s, p) }

";"                     { \p s -> (SEMI, p) }
":"                     { \p s -> (COLON, p) }
"/"                     { \p s -> (SLASH, p) }
"-"                     { \p s -> (MINUS, p) }
"."                     { \p s -> (DOT, p) }
"*"                     { \p s -> (ASTERISK, p) }
"["                     { \p s -> (LBRAC, p) }
"]"                     { \p s -> (RBRAC, p) }
"="                     { \p s -> (EQUALS, p) }
")"                     { \p s -> (RPAREN, p) }
.                       { \p s -> (DELIM (head s), p) }


{
type TokenPosn = (Token, AlexPosn)

data Token =
        S               |
        INCLUDES        |
        DASHMATCH       |
        LBRACE          |
        PLUS            |
        GREATER         |
        COMMA           |
        STRING String   |
        IDENT String    |
        HASH String     |
        IMPORT_SYM      |
        PAGE_SYM        |
        MEDIA_SYM       |
        CHARSET_SYM     |
        IMPORTANT_SYM   |
        EMS Float       |
        EXS Float       |
        LENGTH String   |
        ANGLE String    |
        TIME String     |
        FREQ String     |
        DIMEN String    |
        PERCENTAGE Float |
        NUMBER Float    |
        URI String      |
        FUNCTION String |
        SEMI            |
        RBRACE          |
        COLON           |
        SLASH           |
        MINUS           |
        DOT             |
        ASTERISK        |
        LBRAC           |
        RBRAC           |
        EQUALS          |
        RPAREN          |
        DELIM Char
        deriving (Eq, Show)
}
1	james	63	--
2			-- This file is part of FactorCSS
3			-- Licensed under the MIT License,
4			-- http://www.opensource.org/licenses/mit-license
5			-- Copyright 2004 James Bursa <james@semichrome.net>
6			--
7
8	james	61	-- Tokeniser for CSS 2.1
9			-- See CSS 2.1 G.2
10
11			{
12			module Tokeniser where
13			}
14
15	james	65	%wrapper "posn"
16	james	61
17			$h = [0-9a-f]
18			$nonascii = [\x80-\xff]
19			@unicode = \\$h{1,6}(\r\n\|[\ \t\r\n\f])?
20			@escape = @unicode\|\\[\x20-\x7e\x80-\xff]
21	james	65	@nmstart = [_a-zA-Z]\|$nonascii\|@escape
22	james	61	@nmchar = [\-_a-zA-Z0-9]\|$nonascii\|@escape
23			@nl = \n\|\r\n\|\r\|\f
24	james	64	@string1 = \"([\t\ \!\#\x24\x25&\x28-\x7e]\|\\@nl\|\'\|$nonascii\|@escape\|\.)*\"
25			@string2 = \'([\t\ \!\#\x24\x25&\x28-\x7e]\|\\@nl\|\"\|$nonascii\|@escape\|\.)*\'
26	james	61	@ident = @nmstart@nmchar*
27			@name = @nmchar+
28			@num = [\-\053]?[0-9]+\|[0-9]*"."[0-9]+
29			@string = @string1\|@string2
30	james	64	@url = ([\!\#\x24\x25&\x2a-\x7e]\|$nonascii\|@escape)*
31	james	61	$s = [\ \t\r\n\f]
32			@w = $s*
33			@range = \?{1,6}\|$h(\?{0,5}\|$h(\?{0,4}\|$h(\?{0,3}\|$h(\?{0,2}\|$h(\??\|$h)))))
34	james	64	$all = \x00-\xff
35			@comment = \/\$all#\x2a\+($all#[\/\x2a]$all#\x2a\+)\/
36	james	61
37			tokens :-
38
39	james	69	$s+ { \p s -> (S, p) }
40	james	61	@comment ;
41
42			"<!--" ;
43			"-->" ;
44	james	65	"~=" { \p s -> (INCLUDES, p) }
45			"\|=" { \p s -> (DASHMATCH, p) }
46	james	61
47	james	65	@w"{" { \p s -> (LBRACE, p) }
48	james	70	@w"}" { \p s -> (RBRACE, p) }
49	james	65	@w"+" { \p s -> (PLUS, p) }
50			@w">" { \p s -> (GREATER, p) }
51			@w"," { \p s -> (COMMA, p) }
52	james	61
53	james	65	@string { \p s -> (STRING s, p) }
54	james	61
55	james	65	@ident { \p s -> (IDENT s, p) }
56	james	61
57	james	65	"#"@name { \p s -> (HASH s, p) }
58	james	61
59	james	65	"@import" { \p s -> (IMPORT_SYM, p) }
60			"@page" { \p s -> (PAGE_SYM, p) }
61			"@media" { \p s -> (MEDIA_SYM, p) }
62			"@charset" { \p s -> (CHARSET_SYM, p) }
63	james	61
64	james	65	"!"@w"important" { \p s -> (IMPORTANT_SYM, p) }
65	james	61
66	james	65	@num em { \p s -> (EMS (read ('0':(take (length s - 2) s))), p) }
67			@num ex { \p s -> (EXS (read ('0':(take (length s - 2) s))), p) }
68			@num px { \p s -> (LENGTH s, p) }
69			@num cm { \p s -> (LENGTH s, p) }
70			@num mm { \p s -> (LENGTH s, p) }
71			@num in { \p s -> (LENGTH s, p) }
72			@num pt { \p s -> (LENGTH s, p) }
73			@num pc { \p s -> (LENGTH s, p) }
74			@num deg { \p s -> (ANGLE s, p) }
75			@num rad { \p s -> (ANGLE s, p) }
76			@num grad { \p s -> (ANGLE s, p) }
77			@num ms { \p s -> (TIME s, p) }
78			@num s { \p s -> (TIME s, p) }
79			@num Hz { \p s -> (FREQ s, p) }
80			@num kHz { \p s -> (FREQ s, p) }
81			@num @ident { \p s -> (DIMEN s, p) }
82			@num "%" { \p s -> (PERCENTAGE (read ('0':(take (length s - 1) s))), p) }
83			@num { \p s -> (NUMBER (read ('0':s)), p) }
84	james	61
85	james	65	"url("@w@string@w")" { \p s -> (URI s, p) }
86			"url("@w@url@w")" { \p s -> (URI s, p) }
87			@ident"\050" { \p s -> (FUNCTION s, p) }
88	james	61
89	james	65	";" { \p s -> (SEMI, p) }
90			":" { \p s -> (COLON, p) }
91			"/" { \p s -> (SLASH, p) }
92			"-" { \p s -> (MINUS, p) }
93			"." { \p s -> (DOT, p) }
94			"*" { \p s -> (ASTERISK, p) }
95			"[" { \p s -> (LBRAC, p) }
96			"]" { \p s -> (RBRAC, p) }
97			"=" { \p s -> (EQUALS, p) }
98			")" { \p s -> (RPAREN, p) }
99			. { \p s -> (DELIM (head s), p) }
100	james	61
101
102			{
103	james	65	type TokenPosn = (Token, AlexPosn)
104
105	james	61	data Token =
106	james	69	S \|
107	james	61	INCLUDES \|
108			DASHMATCH \|
109			LBRACE \|
110			PLUS \|
111			GREATER \|
112			COMMA \|
113			STRING String \|
114			IDENT String \|
115			HASH String \|
116			IMPORT_SYM \|
117			PAGE_SYM \|
118			MEDIA_SYM \|
119			CHARSET_SYM \|
120			IMPORTANT_SYM \|
121			EMS Float \|
122			EXS Float \|
123			LENGTH String \|
124			ANGLE String \|
125			TIME String \|
126			FREQ String \|
127			DIMEN String \|
128			PERCENTAGE Float \|
129			NUMBER Float \|
130			URI String \|
131			FUNCTION String \|
132			SEMI \|
133			RBRACE \|
134			COLON \|
135			SLASH \|
136			MINUS \|
137			DOT \|
138			ASTERISK \|
139			LBRAC \|
140			RBRAC \|
141			EQUALS \|
142			RPAREN \|
143			DELIM Char
144			deriving (Eq, Show)
145			}