diff --git a/Cargo.toml b/Cargo.toml index 253c6c37..a3071942 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,3 +15,4 @@ license = "MPL-2.0" [dependencies] encoding = "0.2" text_writer = "0.1.1" +matches = "0.1" diff --git a/src/ast.rs b/src/ast.rs deleted file mode 100644 index 4c32fb0e..00000000 --- a/src/ast.rs +++ /dev/null @@ -1,176 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -use std::fmt; -use std::slice; -use std::vec; - - -#[deriving(PartialEq, Show)] -pub struct NumericValue { - pub representation: String, - pub value: f64, - pub int_value: Option, -} - - -#[deriving(PartialEq, Show, Copy)] -pub struct SourceLocation { - pub line: uint, // First line is 1 - pub column: uint, // First character of a line is at column 1 -} - - -pub type Node = (ComponentValue, SourceLocation); // TODO this is not a good name - - -#[deriving(PartialEq, Show)] -pub enum ComponentValue { - // Preserved tokens. - Ident(String), - AtKeyword(String), - Hash(String), - IDHash(String), // Hash that is a valid ID selector. - QuotedString(String), - URL(String), - Delim(char), - Number(NumericValue), - Percentage(NumericValue), - Dimension(NumericValue, String), - UnicodeRange(u32, u32), // (start, end) of range - WhiteSpace, - Colon, // : - Semicolon, // ; - Comma, // , - IncludeMatch, // ~= - DashMatch, // |= - PrefixMatch, // ^= - SuffixMatch, // $= - SubstringMatch, // *= - Column, // || - CDO, // - - // Function - Function(String, Vec), // name, arguments - - // Simple block - ParenthesisBlock(Vec), // (…) - SquareBracketBlock(Vec), // […] - CurlyBracketBlock(Vec), // {…} - - // These are always invalid - BadURL, - BadString, - CloseParenthesis, // ) - CloseSquareBracket, // ] - CloseCurlyBracket, // } -} - - -#[deriving(PartialEq)] -pub struct Declaration { - pub location: SourceLocation, - pub name: String, - pub value: Vec, - pub important: bool, -} - -#[deriving(PartialEq)] -pub struct QualifiedRule { - pub location: SourceLocation, - pub prelude: Vec, - pub block: Vec, -} - -#[deriving(PartialEq)] -pub struct AtRule { - pub location: SourceLocation, - pub name: String, - pub prelude: Vec, - pub block: Option>, -} - -#[deriving(PartialEq)] -pub enum DeclarationListItem { - Declaration(Declaration), - AtRule(AtRule), -} - -#[deriving(PartialEq)] -pub enum Rule { - QualifiedRule(QualifiedRule), - AtRule(AtRule), -} - -#[deriving(PartialEq, Copy)] -pub struct SyntaxError { - pub location: SourceLocation, - pub reason: ErrorReason, -} - -#[deriving(PartialEq, Show, Copy)] -pub enum ErrorReason { - EmptyInput, // Parsing a single "thing", found only whitespace. - ExtraInput, // Found more non-whitespace after parsing a single "thing". - MissingQualifiedRuleBlock, // EOF in a qualified rule prelude, before '{' - InvalidDeclarationSyntax, - InvalidBangImportantSyntax, - // This is meant to be extended -} - -impl fmt::Show for SyntaxError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{}:{} {}", self.location.line, self.location.column, self.reason) - } -} - - -pub trait SkipWhitespaceIterable<'a> { - fn skip_whitespace(self) -> SkipWhitespaceIterator<'a>; -} - -impl<'a> SkipWhitespaceIterable<'a> for &'a [ComponentValue] { - fn skip_whitespace(self) -> SkipWhitespaceIterator<'a> { - SkipWhitespaceIterator{ iter_with_whitespace: self.iter() } - } -} - -#[deriving(Clone)] -pub struct SkipWhitespaceIterator<'a> { - pub iter_with_whitespace: slice::Items<'a, ComponentValue>, -} - -impl<'a> Iterator<&'a ComponentValue> for SkipWhitespaceIterator<'a> { - fn next(&mut self) -> Option<&'a ComponentValue> { - for component_value in self.iter_with_whitespace { - if component_value != &ComponentValue::WhiteSpace { return Some(component_value) } - } - None - } -} - - -pub trait MoveSkipWhitespaceIterable { - fn move_skip_whitespace(self) -> MoveSkipWhitespaceIterator; -} - -impl MoveSkipWhitespaceIterable for Vec { - fn move_skip_whitespace(self) -> MoveSkipWhitespaceIterator { - MoveSkipWhitespaceIterator{ iter_with_whitespace: self.into_iter() } - } -} - -pub struct MoveSkipWhitespaceIterator { - iter_with_whitespace: vec::MoveItems, -} - -impl Iterator for MoveSkipWhitespaceIterator { - fn next(&mut self) -> Option { - for component_value in self.iter_with_whitespace { - if component_value != ComponentValue::WhiteSpace { return Some(component_value) } - } - None - } -} diff --git a/src/color.rs b/src/color.rs index 5872269f..553c3bca 100644 --- a/src/color.rs +++ b/src/color.rs @@ -8,9 +8,7 @@ use std::num::{Float, FloatMath}; use text_writer::{mod, TextWriter}; -use ast::{ComponentValue, SkipWhitespaceIterable}; -use ast::ComponentValue::{Number, Percentage, Function, Ident, Hash, IDHash, Comma}; -use serializer::ToCss; +use super::{Token, Parser, ToCss}; #[deriving(Clone, Copy, PartialEq)] @@ -65,12 +63,17 @@ impl fmt::Show for Color { /// Return `Err(())` on invalid or unsupported value (not a color). impl Color { - pub fn parse(component_value: &ComponentValue) -> Result { - match *component_value { - Hash(ref value) | IDHash(ref value) => parse_color_hash(value.as_slice()), - Ident(ref value) => parse_color_keyword(value.as_slice()), - Function(ref name, ref arguments) - => parse_color_function(name.as_slice(), arguments.as_slice()), + pub fn parse(input: &mut Parser) -> Result { + match try!(input.next()) { + Token::Hash(ref value) | Token::IDHash(ref value) => { + parse_color_hash(value.as_slice()) + } + Token::Ident(ref value) => parse_color_keyword(value.as_slice()), + Token::Function(ref name) => { + input.parse_nested_block(|arguments| { + parse_color_function(name.as_slice(), arguments) + }) + } _ => Err(()) } } @@ -78,164 +81,173 @@ impl Color { #[inline] -pub fn parse_color_keyword(value: &str) -> Result { - let lower_value = value.to_ascii_lower(); - let (r, g, b) = match lower_value.as_slice() { - "black" => (0., 0., 0.), - "silver" => (192., 192., 192.), - "gray" => (128., 128., 128.), - "white" => (255., 255., 255.), - "maroon" => (128., 0., 0.), - "red" => (255., 0., 0.), - "purple" => (128., 0., 128.), - "fuchsia" => (255., 0., 255.), - "green" => (0., 128., 0.), - "lime" => (0., 255., 0.), - "olive" => (128., 128., 0.), - "yellow" => (255., 255., 0.), - "navy" => (0., 0., 128.), - "blue" => (0., 0., 255.), - "teal" => (0., 128., 128.), - "aqua" => (0., 255., 255.), +pub fn parse_color_keyword(ident: &str) -> Result { + macro_rules! rgba { + ($red: expr, $green: expr, $blue: expr) => { + Ok(Color::RGBA(RGBA { + red: $red / 255., + green: $green / 255., + blue: $blue / 255., + alpha: 1., + })) + } + } - "aliceblue" => (240., 248., 255.), - "antiquewhite" => (250., 235., 215.), - "aquamarine" => (127., 255., 212.), - "azure" => (240., 255., 255.), - "beige" => (245., 245., 220.), - "bisque" => (255., 228., 196.), - "blanchedalmond" => (255., 235., 205.), - "blueviolet" => (138., 43., 226.), - "brown" => (165., 42., 42.), - "burlywood" => (222., 184., 135.), - "cadetblue" => (95., 158., 160.), - "chartreuse" => (127., 255., 0.), - "chocolate" => (210., 105., 30.), - "coral" => (255., 127., 80.), - "cornflowerblue" => (100., 149., 237.), - "cornsilk" => (255., 248., 220.), - "crimson" => (220., 20., 60.), - "cyan" => (0., 255., 255.), - "darkblue" => (0., 0., 139.), - "darkcyan" => (0., 139., 139.), - "darkgoldenrod" => (184., 134., 11.), - "darkgray" => (169., 169., 169.), - "darkgreen" => (0., 100., 0.), - "darkgrey" => (169., 169., 169.), - "darkkhaki" => (189., 183., 107.), - "darkmagenta" => (139., 0., 139.), - "darkolivegreen" => (85., 107., 47.), - "darkorange" => (255., 140., 0.), - "darkorchid" => (153., 50., 204.), - "darkred" => (139., 0., 0.), - "darksalmon" => (233., 150., 122.), - "darkseagreen" => (143., 188., 143.), - "darkslateblue" => (72., 61., 139.), - "darkslategray" => (47., 79., 79.), - "darkslategrey" => (47., 79., 79.), - "darkturquoise" => (0., 206., 209.), - "darkviolet" => (148., 0., 211.), - "deeppink" => (255., 20., 147.), - "deepskyblue" => (0., 191., 255.), - "dimgray" => (105., 105., 105.), - "dimgrey" => (105., 105., 105.), - "dodgerblue" => (30., 144., 255.), - "firebrick" => (178., 34., 34.), - "floralwhite" => (255., 250., 240.), - "forestgreen" => (34., 139., 34.), - "gainsboro" => (220., 220., 220.), - "ghostwhite" => (248., 248., 255.), - "gold" => (255., 215., 0.), - "goldenrod" => (218., 165., 32.), - "greenyellow" => (173., 255., 47.), - "grey" => (128., 128., 128.), - "honeydew" => (240., 255., 240.), - "hotpink" => (255., 105., 180.), - "indianred" => (205., 92., 92.), - "indigo" => (75., 0., 130.), - "ivory" => (255., 255., 240.), - "khaki" => (240., 230., 140.), - "lavender" => (230., 230., 250.), - "lavenderblush" => (255., 240., 245.), - "lawngreen" => (124., 252., 0.), - "lemonchiffon" => (255., 250., 205.), - "lightblue" => (173., 216., 230.), - "lightcoral" => (240., 128., 128.), - "lightcyan" => (224., 255., 255.), - "lightgoldenrodyellow" => (250., 250., 210.), - "lightgray" => (211., 211., 211.), - "lightgreen" => (144., 238., 144.), - "lightgrey" => (211., 211., 211.), - "lightpink" => (255., 182., 193.), - "lightsalmon" => (255., 160., 122.), - "lightseagreen" => (32., 178., 170.), - "lightskyblue" => (135., 206., 250.), - "lightslategray" => (119., 136., 153.), - "lightslategrey" => (119., 136., 153.), - "lightsteelblue" => (176., 196., 222.), - "lightyellow" => (255., 255., 224.), - "limegreen" => (50., 205., 50.), - "linen" => (250., 240., 230.), - "magenta" => (255., 0., 255.), - "mediumaquamarine" => (102., 205., 170.), - "mediumblue" => (0., 0., 205.), - "mediumorchid" => (186., 85., 211.), - "mediumpurple" => (147., 112., 219.), - "mediumseagreen" => (60., 179., 113.), - "mediumslateblue" => (123., 104., 238.), - "mediumspringgreen" => (0., 250., 154.), - "mediumturquoise" => (72., 209., 204.), - "mediumvioletred" => (199., 21., 133.), - "midnightblue" => (25., 25., 112.), - "mintcream" => (245., 255., 250.), - "mistyrose" => (255., 228., 225.), - "moccasin" => (255., 228., 181.), - "navajowhite" => (255., 222., 173.), - "oldlace" => (253., 245., 230.), - "olivedrab" => (107., 142., 35.), - "orange" => (255., 165., 0.), - "orangered" => (255., 69., 0.), - "orchid" => (218., 112., 214.), - "palegoldenrod" => (238., 232., 170.), - "palegreen" => (152., 251., 152.), - "paleturquoise" => (175., 238., 238.), - "palevioletred" => (219., 112., 147.), - "papayawhip" => (255., 239., 213.), - "peachpuff" => (255., 218., 185.), - "peru" => (205., 133., 63.), - "pink" => (255., 192., 203.), - "plum" => (221., 160., 221.), - "powderblue" => (176., 224., 230.), - "rebeccapurple" => (102., 51., 153.), - "rosybrown" => (188., 143., 143.), - "royalblue" => (65., 105., 225.), - "saddlebrown" => (139., 69., 19.), - "salmon" => (250., 128., 114.), - "sandybrown" => (244., 164., 96.), - "seagreen" => (46., 139., 87.), - "seashell" => (255., 245., 238.), - "sienna" => (160., 82., 45.), - "skyblue" => (135., 206., 235.), - "slateblue" => (106., 90., 205.), - "slategray" => (112., 128., 144.), - "slategrey" => (112., 128., 144.), - "snow" => (255., 250., 250.), - "springgreen" => (0., 255., 127.), - "steelblue" => (70., 130., 180.), - "tan" => (210., 180., 140.), - "thistle" => (216., 191., 216.), - "tomato" => (255., 99., 71.), - "turquoise" => (64., 224., 208.), - "violet" => (238., 130., 238.), - "wheat" => (245., 222., 179.), - "whitesmoke" => (245., 245., 245.), - "yellowgreen" => (154., 205., 50.), + match_ignore_ascii_case! { ident: + "black" => rgba!(0., 0., 0.), + "silver" => rgba!(192., 192., 192.), + "gray" => rgba!(128., 128., 128.), + "white" => rgba!(255., 255., 255.), + "maroon" => rgba!(128., 0., 0.), + "red" => rgba!(255., 0., 0.), + "purple" => rgba!(128., 0., 128.), + "fuchsia" => rgba!(255., 0., 255.), + "green" => rgba!(0., 128., 0.), + "lime" => rgba!(0., 255., 0.), + "olive" => rgba!(128., 128., 0.), + "yellow" => rgba!(255., 255., 0.), + "navy" => rgba!(0., 0., 128.), + "blue" => rgba!(0., 0., 255.), + "teal" => rgba!(0., 128., 128.), + "aqua" => rgba!(0., 255., 255.), - "transparent" => return Ok(Color::RGBA(RGBA { red: 0., green: 0., blue: 0., alpha: 0. })), - "currentcolor" => return Ok(Color::CurrentColor), - _ => return Err(()), - }; - Ok(Color::RGBA(RGBA { red: r / 255., green: g / 255., blue: b / 255., alpha: 1. })) + "aliceblue" => rgba!(240., 248., 255.), + "antiquewhite" => rgba!(250., 235., 215.), + "aquamarine" => rgba!(127., 255., 212.), + "azure" => rgba!(240., 255., 255.), + "beige" => rgba!(245., 245., 220.), + "bisque" => rgba!(255., 228., 196.), + "blanchedalmond" => rgba!(255., 235., 205.), + "blueviolet" => rgba!(138., 43., 226.), + "brown" => rgba!(165., 42., 42.), + "burlywood" => rgba!(222., 184., 135.), + "cadetblue" => rgba!(95., 158., 160.), + "chartreuse" => rgba!(127., 255., 0.), + "chocolate" => rgba!(210., 105., 30.), + "coral" => rgba!(255., 127., 80.), + "cornflowerblue" => rgba!(100., 149., 237.), + "cornsilk" => rgba!(255., 248., 220.), + "crimson" => rgba!(220., 20., 60.), + "cyan" => rgba!(0., 255., 255.), + "darkblue" => rgba!(0., 0., 139.), + "darkcyan" => rgba!(0., 139., 139.), + "darkgoldenrod" => rgba!(184., 134., 11.), + "darkgray" => rgba!(169., 169., 169.), + "darkgreen" => rgba!(0., 100., 0.), + "darkgrey" => rgba!(169., 169., 169.), + "darkkhaki" => rgba!(189., 183., 107.), + "darkmagenta" => rgba!(139., 0., 139.), + "darkolivegreen" => rgba!(85., 107., 47.), + "darkorange" => rgba!(255., 140., 0.), + "darkorchid" => rgba!(153., 50., 204.), + "darkred" => rgba!(139., 0., 0.), + "darksalmon" => rgba!(233., 150., 122.), + "darkseagreen" => rgba!(143., 188., 143.), + "darkslateblue" => rgba!(72., 61., 139.), + "darkslategray" => rgba!(47., 79., 79.), + "darkslategrey" => rgba!(47., 79., 79.), + "darkturquoise" => rgba!(0., 206., 209.), + "darkviolet" => rgba!(148., 0., 211.), + "deeppink" => rgba!(255., 20., 147.), + "deepskyblue" => rgba!(0., 191., 255.), + "dimgray" => rgba!(105., 105., 105.), + "dimgrey" => rgba!(105., 105., 105.), + "dodgerblue" => rgba!(30., 144., 255.), + "firebrick" => rgba!(178., 34., 34.), + "floralwhite" => rgba!(255., 250., 240.), + "forestgreen" => rgba!(34., 139., 34.), + "gainsboro" => rgba!(220., 220., 220.), + "ghostwhite" => rgba!(248., 248., 255.), + "gold" => rgba!(255., 215., 0.), + "goldenrod" => rgba!(218., 165., 32.), + "greenyellow" => rgba!(173., 255., 47.), + "grey" => rgba!(128., 128., 128.), + "honeydew" => rgba!(240., 255., 240.), + "hotpink" => rgba!(255., 105., 180.), + "indianred" => rgba!(205., 92., 92.), + "indigo" => rgba!(75., 0., 130.), + "ivory" => rgba!(255., 255., 240.), + "khaki" => rgba!(240., 230., 140.), + "lavender" => rgba!(230., 230., 250.), + "lavenderblush" => rgba!(255., 240., 245.), + "lawngreen" => rgba!(124., 252., 0.), + "lemonchiffon" => rgba!(255., 250., 205.), + "lightblue" => rgba!(173., 216., 230.), + "lightcoral" => rgba!(240., 128., 128.), + "lightcyan" => rgba!(224., 255., 255.), + "lightgoldenrodyellow" => rgba!(250., 250., 210.), + "lightgray" => rgba!(211., 211., 211.), + "lightgreen" => rgba!(144., 238., 144.), + "lightgrey" => rgba!(211., 211., 211.), + "lightpink" => rgba!(255., 182., 193.), + "lightsalmon" => rgba!(255., 160., 122.), + "lightseagreen" => rgba!(32., 178., 170.), + "lightskyblue" => rgba!(135., 206., 250.), + "lightslategray" => rgba!(119., 136., 153.), + "lightslategrey" => rgba!(119., 136., 153.), + "lightsteelblue" => rgba!(176., 196., 222.), + "lightyellow" => rgba!(255., 255., 224.), + "limegreen" => rgba!(50., 205., 50.), + "linen" => rgba!(250., 240., 230.), + "magenta" => rgba!(255., 0., 255.), + "mediumaquamarine" => rgba!(102., 205., 170.), + "mediumblue" => rgba!(0., 0., 205.), + "mediumorchid" => rgba!(186., 85., 211.), + "mediumpurple" => rgba!(147., 112., 219.), + "mediumseagreen" => rgba!(60., 179., 113.), + "mediumslateblue" => rgba!(123., 104., 238.), + "mediumspringgreen" => rgba!(0., 250., 154.), + "mediumturquoise" => rgba!(72., 209., 204.), + "mediumvioletred" => rgba!(199., 21., 133.), + "midnightblue" => rgba!(25., 25., 112.), + "mintcream" => rgba!(245., 255., 250.), + "mistyrose" => rgba!(255., 228., 225.), + "moccasin" => rgba!(255., 228., 181.), + "navajowhite" => rgba!(255., 222., 173.), + "oldlace" => rgba!(253., 245., 230.), + "olivedrab" => rgba!(107., 142., 35.), + "orange" => rgba!(255., 165., 0.), + "orangered" => rgba!(255., 69., 0.), + "orchid" => rgba!(218., 112., 214.), + "palegoldenrod" => rgba!(238., 232., 170.), + "palegreen" => rgba!(152., 251., 152.), + "paleturquoise" => rgba!(175., 238., 238.), + "palevioletred" => rgba!(219., 112., 147.), + "papayawhip" => rgba!(255., 239., 213.), + "peachpuff" => rgba!(255., 218., 185.), + "peru" => rgba!(205., 133., 63.), + "pink" => rgba!(255., 192., 203.), + "plum" => rgba!(221., 160., 221.), + "powderblue" => rgba!(176., 224., 230.), + "rebeccapurple" => rgba!(102., 51., 153.), + "rosybrown" => rgba!(188., 143., 143.), + "royalblue" => rgba!(65., 105., 225.), + "saddlebrown" => rgba!(139., 69., 19.), + "salmon" => rgba!(250., 128., 114.), + "sandybrown" => rgba!(244., 164., 96.), + "seagreen" => rgba!(46., 139., 87.), + "seashell" => rgba!(255., 245., 238.), + "sienna" => rgba!(160., 82., 45.), + "skyblue" => rgba!(135., 206., 235.), + "slateblue" => rgba!(106., 90., 205.), + "slategray" => rgba!(112., 128., 144.), + "slategrey" => rgba!(112., 128., 144.), + "snow" => rgba!(255., 250., 250.), + "springgreen" => rgba!(0., 255., 127.), + "steelblue" => rgba!(70., 130., 180.), + "tan" => rgba!(210., 180., 140.), + "thistle" => rgba!(216., 191., 216.), + "tomato" => rgba!(255., 99., 71.), + "turquoise" => rgba!(64., 224., 208.), + "violet" => rgba!(238., 130., 238.), + "wheat" => rgba!(245., 222., 179.), + "whitesmoke" => rgba!(245., 245., 245.), + "yellowgreen" => rgba!(154., 205., 50.), + + "transparent" => Ok(Color::RGBA(RGBA { red: 0., green: 0., blue: 0., alpha: 0. })), + "currentcolor" => Ok(Color::CurrentColor) + _ => Err(()) + } } @@ -278,70 +290,44 @@ fn parse_color_hash(value: &str) -> Result { #[inline] -fn parse_color_function(name: &str, arguments: &[ComponentValue]) - -> Result { - let lower_name = name.to_ascii_lower(); - let lower_name = lower_name.as_slice(); - - let (is_rgb, has_alpha) = - if "rgba" == lower_name { (true, true) } - else if "rgb" == lower_name { (true, false) } - else if "hsl" == lower_name { (false, false) } - else if "hsla" == lower_name { (false, true) } - else { return Err(()) }; - - let mut iter = arguments.skip_whitespace(); - macro_rules! expect_comma( - () => ( match iter.next() { Some(&Comma) => {}, _ => { return Err(()) } } ); - ); - macro_rules! expect_percentage( - () => ( match iter.next() { - Some(&Percentage(ref v)) => v.value, - _ => return Err(()), - }); - ); - macro_rules! expect_integer( - () => ( match iter.next() { - Some(&Number(ref v)) if v.int_value.is_some() => v.value, - _ => return Err(()), - }); - ); - macro_rules! expect_number( - () => ( match iter.next() { - Some(&Number(ref v)) => v.value, - _ => return Err(()), - }); - ); +fn parse_color_function(name: &str, arguments: &mut Parser) -> Result { + let (is_rgb, has_alpha) = match_ignore_ascii_case! { name: + "rgba" => (true, true), + "rgb" => (true, false), + "hsl" => (false, false), + "hsla" => (false, true) + _ => return Err(()) + }; let red: f32; let green: f32; let blue: f32; if is_rgb { // Either integers or percentages, but all the same type. - match iter.next() { - Some(&Number(ref v)) if v.int_value.is_some() => { + match try!(arguments.next()) { + Token::Number(ref v) if v.int_value.is_some() => { red = (v.value / 255.) as f32; - expect_comma!(); - green = (expect_integer!() / 255.) as f32; - expect_comma!(); - blue = (expect_integer!() / 255.) as f32; + try!(arguments.expect_comma()); + green = try!(arguments.expect_integer()) as f32 / 255.; + try!(arguments.expect_comma()); + blue = try!(arguments.expect_integer()) as f32 / 255.; } - Some(&Percentage(ref v)) => { - red = (v.value / 100.) as f32; - expect_comma!(); - green = (expect_percentage!() / 100.) as f32; - expect_comma!(); - blue = (expect_percentage!() / 100.) as f32; + Token::Percentage(ref v) => { + red = v.unit_value as f32; + try!(arguments.expect_comma()); + green = try!(arguments.expect_percentage()) as f32; + try!(arguments.expect_comma()); + blue = try!(arguments.expect_percentage()) as f32; } _ => return Err(()) }; } else { - let hue = expect_number!() / 360.; + let hue = try!(arguments.expect_number()) / 360.; let hue = hue - hue.floor(); - expect_comma!(); - let saturation = (expect_percentage!() / 100.).max(0.).min(1.); - expect_comma!(); - let lightness = (expect_percentage!() / 100.).max(0.).min(1.); + try!(arguments.expect_comma()); + let saturation = (try!(arguments.expect_percentage())).max(0.).min(1.); + try!(arguments.expect_comma()); + let lightness = (try!(arguments.expect_percentage())).max(0.).min(1.); // http://www.w3.org/TR/css3-color/#hsl-color fn hue_to_rgb(m1: f64, m2: f64, mut h: f64) -> f64 { @@ -362,14 +348,11 @@ fn parse_color_function(name: &str, arguments: &[ComponentValue]) } let alpha = if has_alpha { - expect_comma!(); - (expect_number!()).max(0.).min(1.) as f32 + try!(arguments.expect_comma()); + (try!(arguments.expect_number())).max(0.).min(1.) as f32 } else { 1. }; - if iter.next().is_none() { - Ok(Color::RGBA(RGBA { red: red, green: green, blue: blue, alpha: alpha })) - } else { - Err(()) - } + try!(arguments.expect_exhausted()); + Ok(Color::RGBA(RGBA { red: red, green: green, blue: blue, alpha: alpha })) } diff --git a/src/css-parsing-tests/An+B.json b/src/css-parsing-tests/An+B.json index 3f4e89b2..f17849ea 100644 --- a/src/css-parsing-tests/An+B.json +++ b/src/css-parsing-tests/An+B.json @@ -54,12 +54,15 @@ "- 14N-1 ", null, "3.1n-1", null, "3 n-1", null, +"3n-1foo", null, " n-1", [1, -1], " +n-1", [1, -1], " -n-1", [-1, -1], "+ n-1", null, "- n-1", null, +" +n-1foo", null, +" -n-1foo", null, "3N +1", [3, 1], @@ -69,6 +72,8 @@ "- 14n +1 ", null, "3.1N +1", null, "3 n +1", null, +"3n foo", null, +"3n + foo", null, " n +1", [1, 1], " +N +1", [1, 1], diff --git a/src/css-parsing-tests/component_value_list.json b/src/css-parsing-tests/component_value_list.json index 01c9cbb1..70dc06d9 100644 --- a/src/css-parsing-tests/component_value_list.json +++ b/src/css-parsing-tests/component_value_list.json @@ -22,14 +22,15 @@ ["ident", "red--"], ">" ], -"red0 -red --red -\\-red\\ blue 0red -0red \u0000red _Red .red rêd r\\êd \u007F\u0080\u0081", [ +"\\- red0 -red --red -\\-red\\ blue 0red -0red \u0000\\\u0000red _Red .red rêd r\\êd \u007F\u0080\u0081", [ + ["ident", "-"], " ", ["ident", "red0"], " ", ["ident", "-red"], " ", - "-", ["ident", "-red"], " ", + ["ident", "--red"], " ", ["ident", "--red blue"], " ", ["dimension", "0", 0, "integer", "red"], " ", ["dimension", "-0", 0, "integer", "red"], " ", - ["ident", "\uFFFDred"], " ", + ["ident", "\uFFFD\uFFFDred"], " ", ["ident", "_Red"], " ", ".", ["ident", "red"], " ", ["ident", "rêd"], " ", @@ -54,7 +55,7 @@ "rgba0() -rgba() --rgba() -\\-rgba() 0rgba() -0rgba() _rgba() .rgba() rgbâ() \\30rgba() rgba () @rgba() #rgba()", [ ["function", "rgba0"], " ", ["function", "-rgba"], " ", - "-", ["function", "-rgba"], " ", + ["function", "--rgba"], " ", ["function", "--rgba"], " ", ["dimension", "0", 0, "integer", "rgba"], ["()"], " ", ["dimension", "-0", 0, "integer", "rgba"], ["()"], " ", @@ -70,7 +71,7 @@ "@media0 @-Media @--media @-\\-media @0media @-0media @_media @.media @medİa @\\30 media\\", [ ["at-keyword", "media0"], " ", ["at-keyword", "-Media"], " ", - "@", "-", ["ident", "-media"], " ", + ["at-keyword", "--media"], " ", ["at-keyword", "--media"], " ", "@", ["dimension", "0", 0, "integer", "media"], " ", "@", ["dimension", "-0", 0, "integer", "media"], " ", @@ -80,16 +81,17 @@ ["at-keyword", "0media\uFFFD"] ], -"#red0 #-Red #--red #-\\-red #0red #-0red #_Red #.red #rêd #\\.red\\", [ +"#red0 #-Red #--red #-\\-red #0red #-0red #_Red #.red #rêd #êrd #\\.red\\", [ ["hash", "red0", "id"], " ", ["hash", "-Red", "id"], " ", - ["hash", "--red", "unrestricted"], " ", + ["hash", "--red", "id"], " ", ["hash", "--red", "id"], " ", ["hash", "0red", "unrestricted"], " ", ["hash", "-0red", "unrestricted"], " ", ["hash", "_Red", "id"], " ", "#", ".", ["ident", "red"], " ", ["hash", "rêd", "id"], " ", + ["hash", "êrd", "id"], " ", ["hash", ".red\uFFFD", "id"] ], @@ -120,12 +122,12 @@ ["string", "Ͷ76Ͷ76"] ], -"url( '') url('Lorem \"îpsum\"'\n) url('a\\\nb' ) url('a\nb' \\){ ) url('eof", [ - ["url", ""], " ", - ["url", "Lorem \"îpsum\""], " ", - ["url", "ab"], " ", - ["error", "bad-url"], " ", - ["url", "eof"] +"url( '') url('Lorem \"îpsum\"'\n) url('a\\\nb' ) url('a\nb) url('eof", [ + ["function", "url", " ", ["string", ""]], " ", + ["function", "url", ["string", "Lorem \"îpsum\""], " "], " ", + ["function", "url", ["string", "ab"], " "], " ", + ["function", "url", ["error", "bad-string"], " ", ["ident", "b"]], " ", + ["function", "url", ["string", "eof"]] ], "url(", [ @@ -136,17 +138,17 @@ ["url", ""] ], -"url(\"\") url(\"Lorem 'îpsum'\"\n) url(\"a\\\nb\" ) url(\"a\nb\" \\){ ) url(\"eof", [ - ["url", ""], " ", - ["url", "Lorem 'îpsum'"], " ", - ["url", "ab"], " ", - ["error", "bad-url"], " ", - ["url", "eof"] +"url(\"\") url(\"Lorem 'îpsum'\"\n) url(\"a\\\nb\" ) url(\"a\nb) url(\"eof", [ + ["function", "url", ["string", ""]], " ", + ["function", "url", ["string", "Lorem 'îpsum'"], " "], " ", + ["function", "url", ["string", "ab"], " "], " ", + ["function", "url", ["error", "bad-string"], " ", ["ident", "b"]], " ", + ["function", "url", ["string", "eof"]] ], "url(\"Lo\\rem \\130 ps\\u m\") url('\\376\\37 6\\000376\\0000376\\", [ - ["url", "Lorem İpsu m"], " ", - ["url", "Ͷ76Ͷ76"] + ["function", "url", ["string", "Lorem İpsu m"]], " ", + ["function", "url", ["string", "Ͷ76Ͷ76"]] ], "URL(foo) Url(foo) ûrl(foo) url (foo) url\\ (foo) url(\t 'foo' ", [ @@ -155,12 +157,18 @@ ["function", "ûrl", ["ident", "foo"]], " ", ["ident", "url"], " ", ["()", ["ident", "foo"]], " ", ["function", "url ", ["ident", "foo"]], " ", - ["url", "foo"] + ["function", "url", " ", ["string", "foo"], " "] ], -"url('a' b) url('c' d)", [["error", "bad-url"], " ", ["error", "bad-url"]], +"url('a' b) url('c' d)", [ + ["function", "url", ["string", "a"], " ", ["ident", "b"]], " ", + ["function", "url", ["string", "c"], " ", ["ident", "d"]] +], -"url('a\nb') url('c\n", [["error", "bad-url"], " ", ["error", "bad-url"]], +"url('a\nb) url('c\n", [ + ["function", "url", ["error", "bad-string"], " ", ["ident", "b"]], " ", + ["function", "url", ["error", "bad-string"], " "] +], "url() url( \t) url(\n Foô\\030\n!\n) url(\na\nb\n) url(a\\ b) url(a(b) url(a\\(b) url(a'b) url(a\\'b) url(a\"b) url(a\\\"b) url(a\nb) url(a\\\nb) url(a\\a b) url(a\\", [ ["url", ""], " ", @@ -219,24 +227,24 @@ ["number", "12", 12, "integer"], " ", ["number", "+34", 34, "integer"], " ", ["number", "-45", -45, "integer"], " ", - ["number", ".67", 0.67, "number"], " ", - ["number", "+.89", 0.89, "number"], " ", - ["number", "-.01", -0.01, "number"], " ", + ["number", "0.67", 0.67, "number"], " ", + ["number", "+0.89", 0.89, "number"], " ", + ["number", "-0.01", -0.01, "number"], " ", ["number", "2.3", 2.3, "number"], " ", ["number", "+45.0", 45, "number"], " ", ["number", "-0.67", -0.67, "number"] ], "12e2 +34e+1 -45E-0 .68e+3 +.79e-1 -.01E2 2.3E+1 +45.0e6 -0.67e0", [ - ["number", "12e2", 1200, "number"], " ", - ["number", "+34e+1", 340, "number"], " ", - ["number", "-45E-0", -45, "number"], " ", - ["number", ".68e+3", 680, "number"], " ", - ["number", "+.79e-1", 0.079, "number"], " ", - ["number", "-.01E2", -1, "number"], " ", - ["number", "2.3E+1", 23, "number"], " ", - ["number", "+45.0e6", 45000000, "number"], " ", - ["number", "-0.67e0", -0.67, "number"] + ["number", "1200.0", 1200, "number"], " ", + ["number", "+340.0", 340, "number"], " ", + ["number", "-45.0", -45, "number"], " ", + ["number", "680.0", 680, "number"], " ", + ["number", "+0.079", 0.079, "number"], " ", + ["number", "-1.0", -1, "number"], " ", + ["number", "23.0", 23, "number"], " ", + ["number", "+45000000.0", 45000000, "number"], " ", + ["number", "-0.67", -0.67, "number"] ], "3. /* Decimal point must have following digits */", [ @@ -248,32 +256,32 @@ ], "3e-2.1 /* Integer exponents only */", [ - ["number", "3e-2", 0.03, "number"], - ["number", ".1", 0.1, "number"], " " + ["number", "0.03", 0.03, "number"], + ["number", "0.1", 0.1, "number"], " " ], "12% +34% -45% .67% +.89% -.01% 2.3% +45.0% -0.67%", [ ["percentage", "12", 12, "integer"], " ", ["percentage", "+34", 34, "integer"], " ", ["percentage", "-45", -45, "integer"], " ", - ["percentage", ".67", 0.67, "number"], " ", - ["percentage", "+.89", 0.89, "number"], " ", - ["percentage", "-.01", -0.01, "number"], " ", + ["percentage", "0.67", 0.67, "number"], " ", + ["percentage", "+0.89", 0.89, "number"], " ", + ["percentage", "-0.01", -0.01, "number"], " ", ["percentage", "2.3", 2.3, "number"], " ", ["percentage", "+45.0", 45, "number"], " ", ["percentage", "-0.67", -0.67, "number"] ], "12e2% +34e+1% -45E-0% .68e+3% +.79e-1% -.01E2% 2.3E+1% +45.0e6% -0.67e0%", [ - ["percentage", "12e2", 1200, "number"], " ", - ["percentage", "+34e+1", 340, "number"], " ", - ["percentage", "-45E-0", -45, "number"], " ", - ["percentage", ".68e+3", 680, "number"], " ", - ["percentage", "+.79e-1", 0.079, "number"], " ", - ["percentage", "-.01E2", -1, "number"], " ", - ["percentage", "2.3E+1", 23, "number"], " ", - ["percentage", "+45.0e6", 45000000, "number"], " ", - ["percentage", "-0.67e0", -0.67, "number"] + ["percentage", "1200.0", 1200, "number"], " ", + ["percentage", "+340.0", 340, "number"], " ", + ["percentage", "-45.0", -45, "number"], " ", + ["percentage", "680.0", 680, "number"], " ", + ["percentage", "+0.079", 0.079, "number"], " ", + ["percentage", "-1.0", -1, "number"], " ", + ["percentage", "23.0", 23, "number"], " ", + ["percentage", "+45000000.0", 45000000, "number"], " ", + ["percentage", "-0.67", -0.67, "number"] ], "12\\% /* Percent sign can not be escaped */", [ @@ -284,30 +292,30 @@ ["dimension", "12", 12, "integer", "px"], " ", ["dimension", "+34", 34, "integer", "px"], " ", ["dimension", "-45", -45, "integer", "px"], " ", - ["dimension", ".67", 0.67, "number", "px"], " ", - ["dimension", "+.89", 0.89, "number", "px"], " ", - ["dimension", "-.01", -0.01, "number", "px"], " ", + ["dimension", "0.67", 0.67, "number", "px"], " ", + ["dimension", "+0.89", 0.89, "number", "px"], " ", + ["dimension", "-0.01", -0.01, "number", "px"], " ", ["dimension", "2.3", 2.3, "number", "px"], " ", ["dimension", "+45.0", 45, "number", "px"], " ", ["dimension", "-0.67", -0.67, "number", "px"] ], "12e2px +34e+1px -45E-0px .68e+3px +.79e-1px -.01E2px 2.3E+1px +45.0e6px -0.67e0px", [ - ["dimension", "12e2", 1200, "number", "px"], " ", - ["dimension", "+34e+1", 340, "number", "px"], " ", - ["dimension", "-45E-0", -45, "number", "px"], " ", - ["dimension", ".68e+3", 680, "number", "px"], " ", - ["dimension", "+.79e-1", 0.079, "number", "px"], " ", - ["dimension", "-.01E2", -1, "number", "px"], " ", - ["dimension", "2.3E+1", 23, "number", "px"], " ", - ["dimension", "+45.0e6", 45000000, "number", "px"], " ", - ["dimension", "-0.67e0", -0.67, "number", "px"] + ["dimension", "1200.0", 1200, "number", "px"], " ", + ["dimension", "+340.0", 340, "number", "px"], " ", + ["dimension", "-45.0", -45, "number", "px"], " ", + ["dimension", "680.0", 680, "number", "px"], " ", + ["dimension", "+0.079", 0.079, "number", "px"], " ", + ["dimension", "-1.0", -1, "number", "px"], " ", + ["dimension", "23.0", 23, "number", "px"], " ", + ["dimension", "+45000000.0", 45000000, "number", "px"], " ", + ["dimension", "-0.67", -0.67, "number", "px"] ], "12red0 12.0-red 12--red 12-\\-red 120red 12-0red 12\u0000red 12_Red 12.red 12rêd", [ ["dimension", "12", 12, "integer", "red0"], " ", ["dimension", "12.0", 12, "number", "-red"], " ", - ["number", "12", 12, "integer"], "-", ["ident", "-red"], " ", + ["dimension", "12", 12, "integer", "--red"], " ", ["dimension", "12", 12, "integer", "--red"], " ", ["dimension", "120", 120, "integer", "red"], " ", ["number", "12", 12, "integer"], ["dimension", "-0", 0, "integer", "red"], " ", @@ -391,7 +399,7 @@ ], "~=|=^=$=*=|| |/**/| ~/**/=", [ - "~=", "|=", "^=", "$=", "*=", "||", "", + "~=", "|=", "^=", "$=", "*=", "||", "")), - - &Function(ref name, ref arguments) => { + Token::WhiteSpace(content) => try!(dest.write_str(content)), + Token::Comment(content) => try!(write!(dest, "/*{}*/", content)), + Token::Colon => try!(dest.write_char(':')), + Token::Semicolon => try!(dest.write_char(';')), + Token::Comma => try!(dest.write_char(',')), + Token::IncludeMatch => try!(dest.write_str("~=")), + Token::DashMatch => try!(dest.write_str("|=")), + Token::PrefixMatch => try!(dest.write_str("^=")), + Token::SuffixMatch => try!(dest.write_str("$=")), + Token::SubstringMatch => try!(dest.write_str("*=")), + Token::Column => try!(dest.write_str("||")), + Token::CDO => try!(dest.write_str("")), + + Token::Function(ref name) => { try!(serialize_identifier(name.as_slice(), dest)); try!(dest.write_char('(')); - try!(arguments.to_css(dest)); - try!(dest.write_char(')')); - }, - &ParenthesisBlock(ref content) => { - try!(dest.write_char('(')); - try!(content.to_css(dest)); - try!(dest.write_char(')')); - }, - &SquareBracketBlock(ref content) => { - try!(dest.write_char('[')); - try!(content.to_css(dest)); - try!(dest.write_char(']')); }, - &CurlyBracketBlock(ref content) => { - try!(dest.write_char('{')); - try!(content.to_css(dest)); - try!(dest.write_char('}')); - }, - - &BadURL => try!(dest.write_str("url()")), - &BadString => try!(dest.write_str("\"\n")), - &CloseParenthesis => try!(dest.write_char(')')), - &CloseSquareBracket => try!(dest.write_char(']')), - &CloseCurlyBracket => try!(dest.write_char('}')), + Token::ParenthesisBlock => try!(dest.write_char('(')), + Token::SquareBracketBlock => try!(dest.write_char('[')), + Token::CurlyBracketBlock => try!(dest.write_char('{')), + + Token::BadUrl => try!(dest.write_str("url()")), + Token::BadString => try!(dest.write_str("\"\n")), + Token::CloseParenthesis => try!(dest.write_char(')')), + Token::CloseSquareBracket => try!(dest.write_char(']')), + Token::CloseCurlyBracket => try!(dest.write_char('}')), } Ok(()) } @@ -164,7 +192,7 @@ where W:TextWriter { fn serialize_char(c: char, dest: &mut W, is_identifier_start: bool) -> text_writer::Result where W: TextWriter { match c { - '0'...'9' if is_identifier_start => try!(dest.write_str(format!("\\3{} ", c).as_slice())), + '0'...'9' if is_identifier_start => try!(write!(dest, "\\3{} ", c)), '-' if is_identifier_start => try!(dest.write_str("\\-")), '0'...'9' | 'A'...'Z' | 'a'...'z' | '_' | '-' => try!(dest.write_char(c)), _ if c > '\x7F' => try!(dest.write_char(c)), @@ -232,163 +260,3 @@ impl<'a, W> TextWriter for CssStringWriter<'a, W> where W: TextWriter { } } } - - -impl<'a> ToCss for [ComponentValue] { - fn to_css(&self, dest: &mut W) -> text_writer::Result where W: TextWriter { - component_values_to_css(self.iter(), dest) - } -} - -impl<'a> ToCss for [Node] { - fn to_css(&self, dest: &mut W) -> text_writer::Result where W: TextWriter { - let component_values = self.iter().map(|n| match n { &(ref c, _) => c }); - component_values_to_css(component_values, dest) - } -} - -fn component_values_to_css<'a, I, W>(mut iter: I, dest: &mut W) -> text_writer::Result -where I: Iterator<&'a ComponentValue>, W: TextWriter { - let mut previous = match iter.next() { - None => return Ok(()), - Some(first) => { try!(first.to_css(dest)); first } - }; - macro_rules! matches( - ($value:expr, $($pattern:pat)|+) => ( - match $value { $($pattern)|+ => true, _ => false } - ); - ); - // This does not borrow-check: for component_value in iter { - loop { match iter.next() { None => break, Some(component_value) => { - let (a, b) = (previous, component_value); - if ( - matches!(*a, Ident(..) | AtKeyword(..) | Hash(..) | IDHash(..) | - Dimension(..) | Delim('#') | Delim('-') | Number(..)) && - matches!(*b, Ident(..) | Function(..) | URL(..) | BadURL(..) | - Number(..) | Percentage(..) | Dimension(..) | UnicodeRange(..)) - ) || ( - matches!(*a, Ident(..)) && - matches!(*b, ParenthesisBlock(..)) - ) || ( - matches!(*a, Ident(..) | AtKeyword(..) | Hash(..) | IDHash(..) | Dimension(..)) && - matches!(*b, Delim('-') | CDC) - ) || ( - matches!(*a, Delim('#') | Delim('-') | Number(..) | Delim('@')) && - matches!(*b, Ident(..) | Function(..) | URL(..) | BadURL(..)) - ) || ( - matches!(*a, Delim('@')) && - matches!(*b, Ident(..) | Function(..) | URL(..) | BadURL(..) | - UnicodeRange(..) | Delim('-')) - ) || ( - matches!(*a, UnicodeRange(..) | Delim('.') | Delim('+')) && - matches!(*b, Number(..) | Percentage(..) | Dimension(..)) - ) || ( - matches!(*a, UnicodeRange(..)) && - matches!(*b, Ident(..) | Function(..) | Delim('?')) - ) || (match (a, b) { (&Delim(a), &Delim(b)) => matches!((a, b), - ('#', '-') | - ('$', '=') | - ('*', '=') | - ('^', '=') | - ('~', '=') | - ('|', '=') | - ('|', '|') | - ('/', '*') - ), _ => false }) { - try!(dest.write_str("/**/")); - } - // Skip whitespace when '\n' was previously written at the previous iteration. - if !matches!((previous, component_value), (&Delim('\\'), &WhiteSpace)) { - try!(component_value.to_css(dest)); - } - if component_value == &Delim('\\') { - try!(dest.write_char('\n')); - } - previous = component_value; - }}} - Ok(()) -} - - -impl ToCss for Declaration { - fn to_css(&self, dest: &mut W) -> text_writer::Result where W: TextWriter { - try!(dest.write_str(self.name.as_slice())); - try!(dest.write_char(':')); - try!(self.value.to_css(dest)); - Ok(()) - } -} - - -impl ToCss for [Declaration] { - fn to_css(&self, dest: &mut W) -> text_writer::Result where W: TextWriter { - for declaration in self.iter() { - try!(declaration.to_css(dest)); - try!(dest.write_char(';')); - } - Ok(()) - } -} - - -impl ToCss for QualifiedRule { - fn to_css(&self, dest: &mut W) -> text_writer::Result where W: TextWriter { - try!(self.prelude.to_css(dest)); - try!(dest.write_char('{')); - try!(self.block.to_css(dest)); - try!(dest.write_char('}')); - Ok(()) - } -} - - -impl ToCss for AtRule { - fn to_css(&self, dest: &mut W) -> text_writer::Result where W: TextWriter { - try!(dest.write_char('@')); - try!(dest.write_str(self.name.as_slice())); - try!(self.prelude.to_css(dest)); - match self.block { - Some(ref block) => { - try!(dest.write_char('{')); - try!(block.to_css(dest)); - try!(dest.write_char('}')); - } - None => try!(dest.write_char(';')) - } - Ok(()) - } -} - - -impl ToCss for DeclarationListItem { - fn to_css(&self, dest: &mut W) -> text_writer::Result where W: TextWriter { - match self { - &DeclarationListItem::Declaration(ref declaration) => declaration.to_css(dest), - &DeclarationListItem::AtRule(ref at_rule) => at_rule.to_css(dest), - } - } -} - - -impl ToCss for [DeclarationListItem] { - fn to_css(&self, dest: &mut W) -> text_writer::Result where W: TextWriter { - for item in self.iter() { - try!(item.to_css(dest)); - match item { - &DeclarationListItem::AtRule(_) => {} - &DeclarationListItem::Declaration(_) => try!(dest.write_char(';')) - } - } - Ok(()) - } -} - - -impl ToCss for Rule { - fn to_css(&self, dest: &mut W) -> text_writer::Result where W: TextWriter { - match self { - &Rule::QualifiedRule(ref rule) => rule.to_css(dest), - &Rule::AtRule(ref rule) => rule.to_css(dest), - } - } -} diff --git a/src/tests.rs b/src/tests.rs index 6e50a81d..b4465b75 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -2,24 +2,26 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -use std::io; -use std::io::{File, Command, Writer, TempDir, IoResult}; +use std::borrow::Cow::Borrowed; +use std::io::{mod, File, Command, Writer, TempDir, IoResult}; use std::num::Float; +use std::mem; use serialize::json::{mod, Json, ToJson}; use test; use encoding::label::encoding_from_whatwg_label; -use super::*; -use ast::*; -use ast::ComponentValue::*; +use super::{Parser, Token, NumericValue, PercentageValue, SourceLocation, + DeclarationListParser, DeclarationParser, RuleListParser, + AtRuleType, AtRuleParser, QualifiedRuleParser, + parse_one_declaration, parse_one_rule, parse_important, + parse_stylesheet_rules_from_bytes, + Color, RGBA, parse_color_keyword, parse_nth, ToCss}; -macro_rules! JString { - ($e: expr) => { Json::String($e.to_string()) } -} macro_rules! JArray { - ($($e: expr),*) => { Json::Array(vec!( $($e),* )) } + ($($e: expr,)*) => { JArray![ $( $e ),* ] }; + ($($e: expr),*) => { Json::Array(vec!( $( $e.to_json() ),* )) } } @@ -61,17 +63,68 @@ fn almost_equals(a: &Json, b: &Json) -> bool { (&Json::Boolean(a), &Json::Boolean(b)) => a == b, (&Json::String(ref a), &Json::String(ref b)) => a == b, - (&Json::Array(ref a), &Json::Array(ref b)) - => a.iter().zip(b.iter()).all(|(ref a, ref b)| almost_equals(*a, *b)), - (&Json::Object(_), &Json::Object(_)) - => panic!("Not implemented"), + (&Json::Array(ref a), &Json::Array(ref b)) => { + a.len() == b.len() && + a.iter().zip(b.iter()).all(|(ref a, ref b)| almost_equals(*a, *b)) + }, + (&Json::Object(_), &Json::Object(_)) => panic!("Not implemented"), (&Json::Null, &Json::Null) => true, _ => false, } } +fn normalize(json: &mut Json) { + match *json { + Json::Array(ref mut list) => { + match find_url(list.as_mut_slice()) { + Some(Ok(url)) => *list = vec!["url".to_json(), Json::String(url)], + Some(Err(())) => *list = vec!["error".to_json(), "bad-url".to_json()], + None => { + for item in list.iter_mut() { + normalize(item) + } + } + } + } + Json::String(ref mut s) => { + if s.as_slice() == "extra-input" || s.as_slice() == "empty" { + *s = "invalid".into_string() + } + } + _ => {} + } +} -fn assert_json_eq(results: Json, expected: Json, message: String) { +fn find_url(list: &mut [Json]) -> Option> { + if let [Json::String(ref a1), Json::String(ref a2), ..] = list.as_mut_slice() { + if !(a1.as_slice() == "function" && a2.as_slice() == "url") { + return None + } + } else { + return None + }; + let args = list.slice_from_mut(2); + + let args = if !args.is_empty() && args[0] == " ".to_json() { + args.slice_from_mut(1) + } else { + args.as_mut_slice() + }; + + if let [Json::Array(ref mut arg1), ref rest..] = args.as_mut_slice() { + if let [Json::String(ref a11), Json::String(ref mut a12)] = arg1.as_mut_slice() { + if a11.as_slice() == "string" && rest.iter().all(|a| a == &" ".to_json()) { + return Some(Ok(mem::replace(a12, String::new()))) + } + } + } + + Some(Err(())) +} + + +fn assert_json_eq(results: json::Json, mut expected: json::Json, message: String) { + normalize(&mut expected); if !almost_equals(&results, &expected) { print_json_diff(&results, &expected).unwrap(); panic!(message) @@ -98,11 +151,11 @@ fn run_raw_json_tests(json_data: &str, run: |Json, Json|) { } -fn run_json_tests(json_data: &str, parse: |input: &str| -> T) { +fn run_json_tests(json_data: &str, parse: |input: &mut Parser| -> Json) { run_raw_json_tests(json_data, |input, expected| { match input { Json::String(input) => { - let result = parse(input.as_slice()).to_json(); + let result = parse(&mut Parser::new(input.as_slice())); assert_json_eq(result, expected, input); }, _ => panic!("Unexpected JSON") @@ -114,7 +167,7 @@ fn run_json_tests(json_data: &str, parse: |input: &str| -> T) { #[test] fn component_value_list() { run_json_tests(include_str!("css-parsing-tests/component_value_list.json"), |input| { - tokenize(input).map(|(c, _)| c).collect::>() + Json::Array(component_values_to_json(input)) }); } @@ -122,7 +175,9 @@ fn component_value_list() { #[test] fn one_component_value() { run_json_tests(include_str!("css-parsing-tests/one_component_value.json"), |input| { - parse_one_component_value(tokenize(input)) + input.parse_entirely(|input| { + Ok(one_component_value_to_json(try!(input.next()), input)) + }).unwrap_or(JArray!["error", "invalid"]) }); } @@ -130,7 +185,9 @@ fn one_component_value() { #[test] fn declaration_list() { run_json_tests(include_str!("css-parsing-tests/declaration_list.json"), |input| { - parse_declaration_list(tokenize(input)).collect::>>() + Json::Array(DeclarationListParser::new(input, JsonParser).map(|result| { + result.unwrap_or(JArray!["error", "invalid"]) + }).collect()) }); } @@ -138,7 +195,7 @@ fn declaration_list() { #[test] fn one_declaration() { run_json_tests(include_str!("css-parsing-tests/one_declaration.json"), |input| { - parse_one_declaration(tokenize(input)) + parse_one_declaration(input, &mut JsonParser).unwrap_or(JArray!["error", "invalid"]) }); } @@ -146,7 +203,9 @@ fn one_declaration() { #[test] fn rule_list() { run_json_tests(include_str!("css-parsing-tests/rule_list.json"), |input| { - parse_rule_list(tokenize(input)).collect::>>() + Json::Array(RuleListParser::new_for_nested_rule(input, JsonParser).map(|result| { + result.unwrap_or(JArray!["error", "invalid"]) + }).collect()) }); } @@ -154,7 +213,9 @@ fn rule_list() { #[test] fn stylesheet() { run_json_tests(include_str!("css-parsing-tests/stylesheet.json"), |input| { - parse_stylesheet_rules(tokenize(input)).collect::>>() + Json::Array(RuleListParser::new_for_stylesheet(input, JsonParser).map(|result| { + result.unwrap_or(JArray!["error", "invalid"]) + }).collect()) }); } @@ -162,7 +223,7 @@ fn stylesheet() { #[test] fn one_rule() { run_json_tests(include_str!("css-parsing-tests/one_rule.json"), |input| { - parse_one_rule(tokenize(input)) + parse_one_rule(input, &mut JsonParser).unwrap_or(JArray!["error", "invalid"]) }); } @@ -170,7 +231,7 @@ fn one_rule() { #[test] fn stylesheet_from_bytes() { run_raw_json_tests(include_str!("css-parsing-tests/stylesheet_bytes.json"), - |input, expected| { + |input, expected| { let map = match input { Json::Object(map) => map, _ => panic!("Unexpected JSON") @@ -185,10 +246,16 @@ fn stylesheet_from_bytes() { let environment_encoding = get_string(&map, "environment_encoding") .and_then(encoding_from_whatwg_label); - let (rules, used_encoding) = parse_stylesheet_rules_from_bytes( - css.as_slice(), protocol_encoding_label, environment_encoding); - - (rules.collect::>(), used_encoding.name().to_string()).to_json() + parse_stylesheet_rules_from_bytes( + css.as_slice(), protocol_encoding_label, environment_encoding, + JsonParser, |encoding, rules| { + Json::Array(vec![ + Json::Array(rules.map(|result| { + result.unwrap_or(JArray!["error", "invalid"]) + }).collect()), + encoding.name().to_json() + ]) + }) }; assert_json_eq(result, expected, Json::Object(map).to_string()); }); @@ -204,25 +271,22 @@ fn stylesheet_from_bytes() { } -fn run_color_tests(json_data: &str, to_json: |result: Option| -> Json) { +fn run_color_tests(json_data: &str, to_json: |result: Result| -> Json) { run_json_tests(json_data, |input| { - match parse_one_component_value(tokenize(input)) { - Ok(component_value) => to_json(Color::parse(&component_value).ok()), - Err(_reason) => Json::Null, - } + to_json(input.parse_entirely(Color::parse)) }); } #[test] fn color3() { - run_color_tests(include_str!("css-parsing-tests/color3.json"), |c| c.to_json()) + run_color_tests(include_str!("css-parsing-tests/color3.json"), |c| c.ok().to_json()) } #[test] fn color3_hsl() { - run_color_tests(include_str!("css-parsing-tests/color3_hsl.json"), |c| c.to_json()) + run_color_tests(include_str!("css-parsing-tests/color3_hsl.json"), |c| c.ok().to_json()) } @@ -231,10 +295,10 @@ fn color3_hsl() { fn color3_keywords() { run_color_tests(include_str!("css-parsing-tests/color3_keywords.json"), |c| { match c { - Some(Color::RGBA(RGBA { red: r, green: g, blue: b, alpha: a })) - => vec!(r * 255., g * 255., b * 255., a).to_json(), - Some(Color::CurrentColor) => JString!("currentColor"), - None => Json::Null, + Ok(Color::RGBA(RGBA { red: r, green: g, blue: b, alpha: a })) + => [r * 255., g * 255., b * 255., a].to_json(), + Ok(Color::CurrentColor) => "currentColor".to_json(), + Err(()) => Json::Null, } }); } @@ -242,29 +306,32 @@ fn color3_keywords() { #[bench] fn bench_color_lookup_red(b: &mut test::Bencher) { - let ident = parse_one_component_value(tokenize("red")).unwrap(); - b.iter(|| assert!(Color::parse(&ident).is_ok())); + b.iter(|| { + test::black_box(parse_color_keyword("red")) + }); } #[bench] fn bench_color_lookup_lightgoldenrodyellow(b: &mut test::Bencher) { - let ident = parse_one_component_value(tokenize("lightgoldenrodyellow")).unwrap(); - b.iter(|| assert!(Color::parse(&ident).is_ok())); + b.iter(|| { + test::black_box(parse_color_keyword("lightgoldenrodyellow")) + }); } #[bench] fn bench_color_lookup_fail(b: &mut test::Bencher) { - let ident = parse_one_component_value(tokenize("lightgoldenrodyellowbazinga")).unwrap(); - b.iter(|| assert!(Color::parse(&ident).is_err())); + b.iter(|| { + test::black_box(parse_color_keyword("lightgoldenrodyellowbazinga")) + }); } #[test] fn nth() { run_json_tests(include_str!("css-parsing-tests/An+B.json"), |input| { - parse_nth(tokenize(input).map(|(c, _)| c).collect::>().as_slice()).ok() + input.parse_entirely(parse_nth).ok().to_json() }); } @@ -272,9 +339,28 @@ fn nth() { #[test] fn serializer() { run_json_tests(include_str!("css-parsing-tests/component_value_list.json"), |input| { - let component_values = tokenize(input).map(|(c, _)| c).collect::>(); - let serialized = component_values.to_css_string(); - tokenize(serialized.as_slice()).map(|(c, _)| c).collect::>() + fn write_to(input: &mut Parser, string: &mut String) { + while let Ok(token) = input.next_including_whitespace_and_comments() { + token.to_css(string).unwrap(); + let closing_token = match token { + Token::Function(_) | Token::ParenthesisBlock => Some(Token::CloseParenthesis), + Token::SquareBracketBlock => Some(Token::CloseSquareBracket), + Token::CurlyBracketBlock => Some(Token::CloseCurlyBracket), + _ => None + }; + if let Some(closing_token) = closing_token { + input.parse_nested_block(|input| { + write_to(input, string); + Ok(()) + }).unwrap(); + closing_token.to_css(string).unwrap(); + } + } + } + let mut serialized = String::new(); + write_to(input, &mut serialized); + let parser = &mut Parser::new(serialized.as_slice()); + Json::Array(component_values_to_json(parser)) }); } @@ -282,212 +368,202 @@ fn serializer() { #[test] fn serialize_current_color() { let c = Color::CurrentColor; - assert!(format!("{}", c).as_slice() == "currentColor"); + assert!(c.to_css_string().as_slice() == "currentColor"); } #[test] fn serialize_rgb_full_alpha() { let c = Color::RGBA(RGBA { red: 1.0, green: 0.9, blue: 0.8, alpha: 1.0 }); - assert!(format!("{}", c).as_slice() == "rgb(255, 230, 204)"); + assert!(c.to_css_string().as_slice() == "rgb(255, 230, 204)"); } #[test] fn serialize_rgba() { let c = Color::RGBA(RGBA { red: 0.1, green: 0.2, blue: 0.3, alpha: 0.5 }); - assert!(format!("{}", c).as_slice() == "rgba(26, 51, 77, 0.5)"); + assert!(c.to_css_string().as_slice() == "rgba(26, 51, 77, 0.5)"); } - -impl ToJson for Result { - fn to_json(&self) -> json::Json { - match *self { - Ok(ref a) => a.to_json(), - Err(ref b) => b.to_json(), - } - } +#[test] +fn line_numbers() { + let mut input = Parser::new("foo bar\nbaz\r\n\n\"a\\\r\nb\""); + assert_eq!(input.current_source_location(), SourceLocation { line: 1, column: 1 }); + assert_eq!(input.next_including_whitespace(), Ok(Token::Ident(Borrowed("foo")))); + assert_eq!(input.current_source_location(), SourceLocation { line: 1, column: 4 }); + assert_eq!(input.next_including_whitespace(), Ok(Token::WhiteSpace(" "))); + assert_eq!(input.current_source_location(), SourceLocation { line: 1, column: 5 }); + assert_eq!(input.next_including_whitespace(), Ok(Token::Ident(Borrowed("bar")))); + assert_eq!(input.current_source_location(), SourceLocation { line: 1, column: 8 }); + assert_eq!(input.next_including_whitespace(), Ok(Token::WhiteSpace("\n"))); + assert_eq!(input.current_source_location(), SourceLocation { line: 2, column: 1 }); + assert_eq!(input.next_including_whitespace(), Ok(Token::Ident(Borrowed("baz")))); + assert_eq!(input.current_source_location(), SourceLocation { line: 2, column: 4 }); + let position = input.position(); + + assert_eq!(input.next_including_whitespace(), Ok(Token::WhiteSpace("\r\n\n"))); + assert_eq!(input.current_source_location(), SourceLocation { line: 4, column: 1 }); + + assert_eq!(input.source_location(position), SourceLocation { line: 2, column: 4 }); + + assert_eq!(input.next_including_whitespace(), Ok(Token::QuotedString(Borrowed("ab")))); + assert_eq!(input.current_source_location(), SourceLocation { line: 5, column: 3 }); + assert_eq!(input.next_including_whitespace(), Err(())); } - -impl ToJson for Result { +impl ToJson for Color { fn to_json(&self) -> json::Json { match *self { - Ok(ref a) => a.to_json(), - Err(ref b) => b.to_json(), + Color::RGBA(RGBA { red, green, blue, alpha }) => { + [red, green, blue, alpha].to_json() + }, + Color::CurrentColor => "currentColor".to_json(), } } } -impl ToJson for Result { - fn to_json(&self) -> json::Json { - match *self { - Ok(ref a) => a.to_json(), - Err(ref b) => b.to_json(), +struct JsonParser; + + +impl DeclarationParser for JsonParser { + fn parse_value(&mut self, name: &str, input: &mut Parser) -> Result { + let mut value = vec![]; + let mut important = false; + loop { + let start_position = input.position(); + if let Ok(mut token) = input.next_including_whitespace() { + // Hack to deal with css-parsing-tests assuming that + // `!important` in the middle of a declaration value is OK. + // This can never happen per spec + // (even CSS Variables forbid top-level `!`) + if token == Token::Delim('!') { + input.reset(start_position); + if parse_important(input).is_ok() { + if input.is_exhausted() { + important = true; + break + } + } + input.reset(start_position); + token = input.next_including_whitespace().unwrap(); + } + value.push(one_component_value_to_json(token, input)); + } else { + break + } } + Ok(JArray![ + "declaration", + name, + value, + important, + ]) } } - -impl ToJson for Result { - fn to_json(&self) -> json::Json { - match *self { - Ok(ref a) => a.to_json(), - Err(ref b) => b.to_json(), - } +impl AtRuleParser, Json> for JsonParser { + fn parse_prelude(&mut self, name: &str, input: &mut Parser) + -> Result, Json>, ()> { + Ok(AtRuleType::OptionalBlock(vec![ + "at-rule".to_json(), + name.to_json(), + Json::Array(component_values_to_json(input)), + ])) } -} - -impl ToJson for SyntaxError { - fn to_json(&self) -> json::Json { - Json::Array(vec!(JString!("error"), JString!(match self.reason { - ErrorReason::EmptyInput => "empty", - ErrorReason::ExtraInput => "extra-input", - _ => "invalid", - }))) + fn parse_block(&mut self, mut prelude: Vec, input: &mut Parser) -> Result { + prelude.push(Json::Array(component_values_to_json(input))); + Ok(Json::Array(prelude)) } -} - -impl ToJson for Color { - fn to_json(&self) -> json::Json { - match *self { - Color::RGBA(RGBA { red: r, green: g, blue: b, alpha: a }) => vec!(r, g, b, a).to_json(), - Color::CurrentColor => JString!("currentColor"), - } + fn rule_without_block(&mut self, mut prelude: Vec) -> Result { + prelude.push(Json::Null); + Ok(Json::Array(prelude)) } } - -impl ToJson for Rule { - fn to_json(&self) -> json::Json { - match *self { - Rule::QualifiedRule(ref rule) => rule.to_json(), - Rule::AtRule(ref rule) => rule.to_json(), - } +impl QualifiedRuleParser, Json> for JsonParser { + fn parse_prelude(&mut self, input: &mut Parser) -> Result, ()> { + Ok(component_values_to_json(input)) } -} - -impl ToJson for DeclarationListItem { - fn to_json(&self) -> json::Json { - match *self { - DeclarationListItem::Declaration(ref declaration) => declaration.to_json(), - DeclarationListItem::AtRule(ref at_rule) => at_rule.to_json(), - } + fn parse_block(&mut self, prelude: Vec, input: &mut Parser) -> Result { + Ok(JArray![ + "qualified rule", + prelude, + component_values_to_json(input), + ]) } } - -fn list_to_json(list: &Vec<(ComponentValue, SourceLocation)>) -> Vec { - list.iter().map(|tuple| { - match *tuple { - (ref c, _) => c.to_json() - } - }).collect() -} - - -impl ToJson for AtRule { - fn to_json(&self) -> json::Json { - match *self { - AtRule{ ref name, ref prelude, ref block, ..} - => Json::Array(vec!(JString!("at-rule"), name.to_json(), - prelude.to_json(), block.as_ref().map(list_to_json).to_json())) - } +fn component_values_to_json(input: &mut Parser) -> Vec { + let mut values = vec![]; + while let Ok(token) = input.next_including_whitespace() { + values.push(one_component_value_to_json(token, input)); } + values } -impl ToJson for QualifiedRule { - fn to_json(&self) -> json::Json { - match *self { - QualifiedRule{ ref prelude, ref block, ..} - => Json::Array(vec!(JString!("qualified rule"), - prelude.to_json(), Json::Array(list_to_json(block)))) - } +fn one_component_value_to_json(token: Token, input: &mut Parser) -> Json { + fn numeric(value: NumericValue) -> Vec { + vec![ + Token::Number(value).to_css_string().to_json(), + match value.int_value { Some(i) => i.to_json(), None => value.value.to_json() }, + match value.int_value { Some(_) => "integer", None => "number" }.to_json() + ] } -} - -impl ToJson for Declaration { - fn to_json(&self) -> json::Json { - match *self { - Declaration{ ref name, ref value, ref important, ..} - => Json::Array(vec!(JString!("declaration"), name.to_json(), - value.to_json(), important.to_json())) - } + fn nested(input: &mut Parser) -> Vec { + input.parse_nested_block(|input| Ok(component_values_to_json(input))).unwrap() } -} - - -impl ToJson for ComponentValue { - fn to_json(&self) -> json::Json { - fn numeric(value: &NumericValue) -> Vec { - match *value { - NumericValue{representation: ref r, value: ref v, int_value: ref i} - => vec!(r.to_json(), v.to_json(), - JString!(match *i { Some(_) => "integer", _ => "number" })) - } - } - match *self { - Ident(ref value) => JArray!(JString!("ident"), value.to_json()), - AtKeyword(ref value) => JArray!(JString!("at-keyword"), value.to_json()), - Hash(ref value) => JArray!(JString!("hash"), value.to_json(), - JString!("unrestricted")), - IDHash(ref value) => JArray!(JString!("hash"), value.to_json(), JString!("id")), - QuotedString(ref value) => JArray!(JString!("string"), value.to_json()), - URL(ref value) => JArray!(JString!("url"), value.to_json()), - Delim('\\') => JString!("\\"), - Delim(value) => Json::String(String::from_char(1, value)), - - Number(ref value) => Json::Array( - vec!(JString!("number")) + numeric(value).as_slice()), - Percentage(ref value) => Json::Array( - vec!(JString!("percentage")) + numeric(value).as_slice()), - Dimension(ref value, ref unit) => Json::Array( - vec!(JString!("dimension")) + numeric(value).as_slice() - + [unit.to_json()].as_slice()), - - UnicodeRange(start, end) - => JArray!(JString!("unicode-range"), start.to_json(), end.to_json()), - - WhiteSpace => JString!(" "), - Colon => JString!(":"), - Semicolon => JString!(";"), - Comma => JString!(","), - IncludeMatch => JString!("~="), - DashMatch => JString!("|="), - PrefixMatch => JString!("^="), - SuffixMatch => JString!("$="), - SubstringMatch => JString!("*="), - Column => JString!("||"), - CDO => JString!(""), - - Function(ref name, ref arguments) - => Json::Array( - vec!(JString!("function"), name.to_json()) - + arguments.iter().map(|a| a.to_json()).collect::>().as_slice()), - ParenthesisBlock(ref content) - => Json::Array( - vec!(JString!("()")) - + content.iter().map(|c| c.to_json()).collect::>().as_slice()), - SquareBracketBlock(ref content) - => Json::Array( - vec!(JString!("[]")) - + content.iter().map(|c| c.to_json()).collect::>().as_slice()), - CurlyBracketBlock(ref content) - => Json::Array(vec!(JString!("{}")) + list_to_json(content).as_slice()), - - BadURL => JArray!(JString!("error"), JString!("bad-url")), - BadString => JArray!(JString!("error"), JString!("bad-string")), - CloseParenthesis => JArray!(JString!("error"), JString!(")")), - CloseSquareBracket => JArray!(JString!("error"), JString!("]")), - CloseCurlyBracket => JArray!(JString!("error"), JString!("}")), - } + match token { + Token::Ident(value) => JArray!["ident", value], + Token::AtKeyword(value) => JArray!["at-keyword", value], + Token::Hash(value) => JArray!["hash", value, "unrestricted"], + Token::IDHash(value) => JArray!["hash", value, "id"], + Token::QuotedString(value) => JArray!["string", value], + Token::Url(value) => JArray!["url", value], + Token::Delim('\\') => "\\".to_json(), + Token::Delim(value) => String::from_char(1, value).to_json(), + + Token::Number(value) => Json::Array(vec!["number".to_json()] + numeric(value)), + Token::Percentage(PercentageValue { unit_value, int_value, signed }) => Json::Array( + vec!["percentage".to_json()] + numeric(NumericValue { + value: unit_value * 100., + int_value: int_value, + signed: signed, + })), + Token::Dimension(value, unit) => Json::Array( + vec!["dimension".to_json()] + numeric(value) + [unit.to_json()].as_slice()), + + Token::UnicodeRange(start, end) => JArray!["unicode-range", start, end], + + Token::WhiteSpace(_) => " ".to_json(), + Token::Comment(_) => "/**/".to_json(), + Token::Colon => ":".to_json(), + Token::Semicolon => ";".to_json(), + Token::Comma => ",".to_json(), + Token::IncludeMatch => "~=".to_json(), + Token::DashMatch => "|=".to_json(), + Token::PrefixMatch => "^=".to_json(), + Token::SuffixMatch => "$=".to_json(), + Token::SubstringMatch => "*=".to_json(), + Token::Column => "||".to_json(), + Token::CDO => "".to_json(), + + Token::Function(name) => Json::Array(vec!["function".to_json(), name.to_json()] + + nested(input)), + Token::ParenthesisBlock => Json::Array(vec!["()".to_json()] + nested(input)), + Token::SquareBracketBlock => Json::Array(vec!["[]".to_json()] + nested(input)), + Token::CurlyBracketBlock => Json::Array(vec!["{}".to_json()] + nested(input)), + Token::BadUrl => JArray!["error", "bad-url"], + Token::BadString => JArray!["error", "bad-string"], + Token::CloseParenthesis => JArray!["error", ")"], + Token::CloseSquareBracket => JArray!["error", "]"], + Token::CloseCurlyBracket => JArray!["error", "}"], } } diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 7950057d..4f3b35f7 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -4,587 +4,752 @@ // http://dev.w3.org/csswg/css3-syntax/#tokenization -use std::{char, num}; +use std::cell::Cell; +use std::char; +use std::num; use std::ascii::AsciiExt; - -use ast::*; -use ast::ComponentValue::*; +use std::borrow::{Cow, ToOwned}; +use std::str::CowString; +use std::borrow::Cow::{Owned, Borrowed}; + +use self::Token::*; + + +#[deriving(PartialEq, Show, Clone)] +pub enum Token<'a> { + // Preserved tokens. + Ident(CowString<'a>), + AtKeyword(CowString<'a>), + Hash(CowString<'a>), + IDHash(CowString<'a>), // Hash that is a valid ID selector. + QuotedString(CowString<'a>), + Url(CowString<'a>), + Delim(char), + Number(NumericValue), + Percentage(PercentageValue), + Dimension(NumericValue, CowString<'a>), + UnicodeRange(u32, u32), // (start, end) of range + WhiteSpace(&'a str), + Comment(&'a str), + Colon, // : + Semicolon, // ; + Comma, // , + IncludeMatch, // ~= + DashMatch, // |= + PrefixMatch, // ^= + SuffixMatch, // $= + SubstringMatch, // *= + Column, // || + CDO, // + + // Function + Function(CowString<'a>), // name + + // Simple block + ParenthesisBlock, // (…) + SquareBracketBlock, // […] + CurlyBracketBlock, // {…} + + // These are always invalid + BadUrl, + BadString, + CloseParenthesis, // ) + CloseSquareBracket, // ] + CloseCurlyBracket, // } +} -/// Returns a `Iterator<(ComponentValue, SourceLocation)>` -pub fn tokenize(input: &str) -> Tokenizer { - let input = preprocess(input); - Tokenizer { - length: input.len(), - input: input, - position: 0, - line: 1, - last_line_start: 0, +impl<'a> Token<'a> { + pub fn into_owned(self) -> Token<'static> { + match self { + Token::Ident(value) => Token::Ident(Cow::Owned(value.into_owned())), + Token::AtKeyword(value) => Token::AtKeyword(Cow::Owned(value.into_owned())), + Token::Hash(value) => Token::Hash(Cow::Owned(value.into_owned())), + Token::IDHash(value) => Token::IDHash(Cow::Owned(value.into_owned())), + Token::QuotedString(value) => Token::QuotedString(Cow::Owned(value.into_owned())), + Token::Url(value) => Token::Url(Cow::Owned(value.into_owned())), + Token::Delim(ch) => Token::Delim(ch), + Token::Number(value) => Token::Number(value), + Token::Percentage(value) => Token::Percentage(value), + Token::Dimension(value, unit) => Token::Dimension(value, Cow::Owned(unit.into_owned())), + Token::UnicodeRange(start, end) => Token::UnicodeRange(start, end), + Token::WhiteSpace(content) => { + Token::WhiteSpace(if matches!(content.as_bytes()[0], b'\n' | b'\r' | b'\x0C') { "\n" } else { " " }) + } + Token::Comment(_) => Token::Comment(""), + Token::Colon => Token::Colon, + Token::Semicolon => Token::Semicolon, + Token::Comma => Token::Comma, + Token::IncludeMatch => Token::IncludeMatch, + Token::DashMatch => Token::DashMatch, + Token::PrefixMatch => Token::PrefixMatch, + Token::SuffixMatch => Token::SuffixMatch, + Token::SubstringMatch => Token::SubstringMatch, + Token::Column => Token::Column, + Token::CDO => Token::CDO, + Token::CDC => Token::CDC, + Token::Function(name) => Token::Function(Cow::Owned(name.into_owned())), + Token::ParenthesisBlock => Token::ParenthesisBlock, + Token::SquareBracketBlock => Token::SquareBracketBlock, + Token::CurlyBracketBlock => Token::CurlyBracketBlock, + Token::BadUrl => Token::BadUrl, + Token::BadString => Token::BadString, + Token::CloseParenthesis => Token::CloseParenthesis, + Token::CloseSquareBracket => Token::CloseSquareBracket, + Token::CloseCurlyBracket => Token::CloseCurlyBracket, + } } } -impl Iterator for Tokenizer { - #[inline] - fn next(&mut self) -> Option { next_component_value(self) } + +#[deriving(PartialEq, Show, Copy, Clone)] +pub struct NumericValue { + pub value: f64, + pub int_value: Option, + /// Whether the number had a `+` or `-` sign. + pub signed: bool, } -// *********** End of public API *********** +#[deriving(PartialEq, Show, Copy, Clone)] +pub struct PercentageValue { + /// This (but not int_value) is divided by 100 + pub unit_value: f64, + pub int_value: Option, + /// Whether the number had a `+` or `-` sign. + pub signed: bool, +} + +#[deriving(Clone)] +pub struct Tokenizer<'a> { + input: &'a str, -#[inline] -fn preprocess(input: &str) -> String { - // Replace: - // "\r\n" => "\n" - // "\r" => "\n" - // "\x0C" => "\n" - // "\x00" => "\u{FFFD}" - - let bytes = input.as_bytes(); - let mut result: Vec = Vec::with_capacity(bytes.len()); - let mut last: u8 = 0; - for byte in bytes.iter() { - match *byte { - b'\n' if last == b'\r' => (), - b'\r' | b'\x0C' => result.push(b'\n'), - b'\0' => result.push_all("\u{FFFD}".as_bytes()), - _ => result.push(*byte), + /// Counted in bytes, not code points. From 0. + position: uint, + + /// Cache for `source_location()` + last_known_line_break: Cell<(uint, uint)>, +} + + +impl<'a> Tokenizer<'a> { + #[inline] + pub fn new(input: &str) -> Tokenizer { + Tokenizer { + input: input, + position: 0, + last_known_line_break: Cell::new((1, 0)), } + } - last = *byte; + #[inline] + pub fn next(&mut self) -> Result, ()> { + next_token(self).ok_or(()) } - unsafe { String::from_utf8_unchecked(result) } -} + #[inline] + pub fn position(&self) -> SourcePosition { + SourcePosition(self.position) + } + #[inline] + pub fn reset(&mut self, new_position: SourcePosition) { + self.position = new_position.0; + } -#[test] -fn test_preprocess() { - assert!("" == preprocess("").as_slice()); - assert!("Lorem\n\n\t\u{FFFD}ipusm\ndoror\u{FFFD}á\n" == - preprocess("Lorem\r\n\n\t\x00ipusm\ndoror\u{FFFD}á\r").as_slice()); -} + #[inline] + pub fn slice_from(&self, start_pos: SourcePosition) -> &'a str { + self.input.slice(start_pos.0, self.position) + } -#[cfg(test)] -mod bench_preprocess { - extern crate test; + #[inline] + pub fn current_source_location(&self) -> SourceLocation { + let position = SourcePosition(self.position); + self.source_location(position) + } - #[bench] - fn bench_preprocess(b: &mut test::Bencher) { - let source = "Lorem\n\t\u{FFFD}ipusm\ndoror\u{FFFD}á\n"; - b.iter(|| { - let _ = super::preprocess(source); - }); + pub fn source_location(&self, position: SourcePosition) -> SourceLocation { + let target = position.0; + let mut line_number; + let mut position; + let (last_known_line_number, position_after_last_known_newline) = + self.last_known_line_break.get(); + if target >= position_after_last_known_newline { + position = position_after_last_known_newline; + line_number = last_known_line_number; + } else { + position = 0; + line_number = 1; + } + let mut source = self.input.slice(position, target); + while let Some(newline_position) = source.find(['\n', '\r', '\x0C'].as_slice()) { + let offset = newline_position + + if source.slice_from(newline_position).starts_with("\r\n") { + 2 + } else { + 1 + }; + source = source.slice_from(offset); + position += offset; + line_number += 1; + } + debug_assert!(position <= target); + self.last_known_line_break.set((line_number, position)); + SourceLocation { + line: line_number, + // `target == position` when `target` is at the beginning of the line, + // so add 1 so that the column numbers start at 1. + column: target - position + 1, + } } -} + #[inline] + pub fn next_byte(&self) -> Option { + if self.is_eof() { + None + } else { + Some(self.input.as_bytes()[self.position]) + } + } -pub struct Tokenizer { - input: String, - length: uint, // All counted in bytes, not characters - position: uint, // All counted in bytes, not characters - line: uint, - last_line_start: uint, // All counted in bytes, not characters -} + // If false, `tokenizer.next_char()` will not panic. + #[inline] + fn is_eof(&self) -> bool { !self.has_at_least(0) } + // If true, the input has at least `n` bytes left *after* the current one. + // That is, `tokenizer.char_at(n)` will not panic. + #[inline] + fn has_at_least(&self, n: uint) -> bool { self.position + n < self.input.len() } -impl Tokenizer { #[inline] - fn is_eof(&self) -> bool { self.position >= self.length } + pub fn advance(&mut self, n: uint) { self.position += n } // Assumes non-EOF #[inline] - fn current_char(&self) -> char { self.char_at(0) } + fn next_char(&self) -> char { self.char_at(0) } #[inline] fn char_at(&self, offset: uint) -> char { - self.input.as_slice().char_at(self.position + offset) + self.input.char_at(self.position + offset) + } + + #[inline] + fn has_newline_at(&self, offset: uint) -> bool { + self.position + offset < self.input.len() && + matches!(self.char_at(offset), '\n' | '\r' | '\x0C') } #[inline] fn consume_char(&mut self) -> char { - let range = self.input.as_slice().char_range_at(self.position); + let range = self.input.char_range_at(self.position); self.position = range.next; range.ch } #[inline] fn starts_with(&self, needle: &str) -> bool { - self.input.as_slice().slice_from(self.position).starts_with(needle) - } - - #[inline] - fn new_line(&mut self) { - if cfg!(test) { - assert!(self.input.as_slice().char_at(self.position - 1) == '\n') - } - self.line += 1; - self.last_line_start = self.position; + self.input.slice_from(self.position).starts_with(needle) } } -macro_rules! is_match { - ($value:expr, $($pattern:pat)|+) => ( - match $value { $($pattern)|+ => true, _ => false } - ); + +#[deriving(PartialEq, Eq, PartialOrd, Ord, Show, Clone, Copy)] +pub struct SourcePosition(uint); + + +#[deriving(PartialEq, Eq, Show, Clone, Copy)] +pub struct SourceLocation { + /// Starts at 1 + pub line: uint, + /// Starts at 1 + pub column: uint, } -fn next_component_value(tokenizer: &mut Tokenizer) -> Option { - consume_comments(tokenizer); +fn next_token<'a>(tokenizer: &mut Tokenizer<'a>) -> Option> { if tokenizer.is_eof() { - if cfg!(test) { - assert!(tokenizer.line == tokenizer.input.as_slice().split('\n').count(), - "The tokenizer is missing a tokenizer.new_line() call somewhere.") - } return None } - let start_location = SourceLocation{ - line: tokenizer.line, - // The start of the line is column 1: - column: tokenizer.position - tokenizer.last_line_start + 1, - }; - let c = tokenizer.current_char(); - let component_value = match c { - '\t' | '\n' | ' ' => { + let c = tokenizer.next_char(); + let token = match c { + '\t' | '\n' | ' ' | '\r' | '\x0C' => { + let start_position = tokenizer.position(); + tokenizer.advance(1); while !tokenizer.is_eof() { - match tokenizer.current_char() { - ' ' | '\t' => tokenizer.position += 1, - '\n' => { - tokenizer.position += 1; - tokenizer.new_line(); - }, + match tokenizer.next_char() { + ' ' | '\t' | '\n' | '\r' | '\x0C' => tokenizer.advance(1), _ => break, } } - WhiteSpace + WhiteSpace(tokenizer.slice_from(start_position)) }, '"' => consume_string(tokenizer, false), '#' => { - tokenizer.position += 1; + tokenizer.advance(1); if is_ident_start(tokenizer) { IDHash(consume_name(tokenizer)) } - else if !tokenizer.is_eof() && match tokenizer.current_char() { + else if !tokenizer.is_eof() && match tokenizer.next_char() { 'a'...'z' | 'A'...'Z' | '0'...'9' | '-' | '_' => true, - '\\' => !tokenizer.starts_with("\\\n"), + '\\' => !tokenizer.has_newline_at(1), _ => c > '\x7F', // Non-ASCII } { Hash(consume_name(tokenizer)) } else { Delim(c) } }, '$' => { - if tokenizer.starts_with("$=") { tokenizer.position += 2; SuffixMatch } - else { tokenizer.position += 1; Delim(c) } + if tokenizer.starts_with("$=") { tokenizer.advance(2); SuffixMatch } + else { tokenizer.advance(1); Delim(c) } }, '\'' => consume_string(tokenizer, true), - '(' => ParenthesisBlock(consume_block(tokenizer, CloseParenthesis)), - ')' => { tokenizer.position += 1; CloseParenthesis }, + '(' => { tokenizer.advance(1); ParenthesisBlock }, + ')' => { tokenizer.advance(1); CloseParenthesis }, '*' => { - if tokenizer.starts_with("*=") { tokenizer.position += 2; SubstringMatch } - else { tokenizer.position += 1; Delim(c) } + if tokenizer.starts_with("*=") { tokenizer.advance(2); SubstringMatch } + else { tokenizer.advance(1); Delim(c) } }, '+' => { if ( - tokenizer.position + 1 < tokenizer.length - && is_match!(tokenizer.char_at(1), '0'...'9') + tokenizer.has_at_least(1) + && matches!(tokenizer.char_at(1), '0'...'9') ) || ( - tokenizer.position + 2 < tokenizer.length + tokenizer.has_at_least(2) && tokenizer.char_at(1) == '.' - && is_match!(tokenizer.char_at(2), '0'...'9') + && matches!(tokenizer.char_at(2), '0'...'9') ) { consume_numeric(tokenizer) } else { - tokenizer.position += 1; + tokenizer.advance(1); Delim(c) } }, - ',' => { tokenizer.position += 1; Comma }, + ',' => { tokenizer.advance(1); Comma }, '-' => { if ( - tokenizer.position + 1 < tokenizer.length - && is_match!(tokenizer.char_at(1), '0'...'9') + tokenizer.has_at_least(1) + && matches!(tokenizer.char_at(1), '0'...'9') ) || ( - tokenizer.position + 2 < tokenizer.length + tokenizer.has_at_least(2) && tokenizer.char_at(1) == '.' - && is_match!(tokenizer.char_at(2), '0'...'9') + && matches!(tokenizer.char_at(2), '0'...'9') ) { consume_numeric(tokenizer) - } else if is_ident_start(tokenizer) { - consume_ident_like(tokenizer) } else if tokenizer.starts_with("-->") { - tokenizer.position += 3; + tokenizer.advance(3); CDC + } else if is_ident_start(tokenizer) { + consume_ident_like(tokenizer) } else { - tokenizer.position += 1; + tokenizer.advance(1); Delim(c) } }, '.' => { - if tokenizer.position + 1 < tokenizer.length - && is_match!(tokenizer.char_at(1), '0'...'9' + if tokenizer.has_at_least(1) + && matches!(tokenizer.char_at(1), '0'...'9' ) { consume_numeric(tokenizer) } else { - tokenizer.position += 1; + tokenizer.advance(1); Delim(c) } } + '/' if tokenizer.starts_with("/*") => { + tokenizer.advance(2); // consume "/*" + let start_position = tokenizer.position(); + let content; + match tokenizer.input.slice_from(tokenizer.position).find_str("*/") { + Some(offset) => { + tokenizer.advance(offset); + content = tokenizer.slice_from(start_position); + tokenizer.advance(2); + } + None => { + tokenizer.position = tokenizer.input.len(); + content = tokenizer.slice_from(start_position); + } + } + Comment(content) + } '0'...'9' => consume_numeric(tokenizer), - ':' => { tokenizer.position += 1; Colon }, - ';' => { tokenizer.position += 1; Semicolon }, + ':' => { tokenizer.advance(1); Colon }, + ';' => { tokenizer.advance(1); Semicolon }, '<' => { if tokenizer.starts_with("