Add unicode support for ID tokens

kpdecker · kpdecker · commit c1020a013097 · 2013-05-27T11:56:28.000-05:00
Fixes handlebars-lang#433 Fixes handlebars-lang#469
diff --git a/dist/handlebars.js b/dist/handlebars.js
@@ -644,7 +644,7 @@ case 33: return 5;
 break;
 }
 };
-lexer.rules = [/^(?:\\\\(?=(\{\{)))/,/^(?:[^\x00]*?(?=(\{\{)))/,/^(?:[^\x00]+)/,/^(?:[^\x00]{2,}?(?=(\{\{|$)))/,/^(?:[\s\S]*?--\}\})/,/^(?:\{\{>)/,/^(?:\{\{#)/,/^(?:\{\{\/)/,/^(?:\{\{\^)/,/^(?:\{\{\s*else\b)/,/^(?:\{\{\{)/,/^(?:\{\{&)/,/^(?:\{\{!--)/,/^(?:\{\{![\s\S]*?\}\})/,/^(?:\{\{)/,/^(?:=)/,/^(?:\.(?=[}/ ]))/,/^(?:\.\.)/,/^(?:[\/.])/,/^(?:\s+)/,/^(?:\}\}\})/,/^(?:\}\})/,/^(?:"(\\["]|[^"])*")/,/^(?:'(\\[']|[^'])*')/,/^(?:@[a-zA-Z]+)/,/^(?:true(?=[}\s]))/,/^(?:false(?=[}\s]))/,/^(?:-?[0-9]+(?=[}\s]))/,/^(?:[a-zA-Z0-9_$:\-]+(?=[=}\s\/.]))/,/^(?:\[[^\]]*\])/,/^(?:.)/,/^(?:\s+)/,/^(?:[a-zA-Z0-9_$\-\/\.]+)/,/^(?:$)/];
+lexer.rules = [/^(?:\\\\(?=(\{\{)))/,/^(?:[^\x00]*?(?=(\{\{)))/,/^(?:[^\x00]+)/,/^(?:[^\x00]{2,}?(?=(\{\{|$)))/,/^(?:[\s\S]*?--\}\})/,/^(?:\{\{>)/,/^(?:\{\{#)/,/^(?:\{\{\/)/,/^(?:\{\{\^)/,/^(?:\{\{\s*else\b)/,/^(?:\{\{\{)/,/^(?:\{\{&)/,/^(?:\{\{!--)/,/^(?:\{\{![\s\S]*?\}\})/,/^(?:\{\{)/,/^(?:=)/,/^(?:\.(?=[}\/ ]))/,/^(?:\.\.)/,/^(?:[\/.])/,/^(?:\s+)/,/^(?:\}\}\})/,/^(?:\}\})/,/^(?:"(\\["]|[^"])*")/,/^(?:'(\\[']|[^'])*')/,/^(?:@[a-zA-Z]+)/,/^(?:true(?=[}\s]))/,/^(?:false(?=[}\s]))/,/^(?:-?[0-9]+(?=[}\s]))/,/^(?:[^\s!"#%-,\.\/;->@\[-\^`\{-~]+(?=[=}\s\/.]))/,/^(?:\[[^\]]*\])/,/^(?:.)/,/^(?:\s+)/,/^(?:[a-zA-Z0-9_$\-\/\.]+)/,/^(?:$)/];
 lexer.conditions = {"mu":{"rules":[5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,33],"inclusive":false},"emu":{"rules":[3],"inclusive":false},"com":{"rules":[4],"inclusive":false},"par":{"rules":[31,32],"inclusive":false},"INITIAL":{"rules":[0,1,2,33],"inclusive":true}};
 return lexer;})()
 parser.lexer = lexer;
diff --git a/release-notes.md b/release-notes.md
@@ -4,6 +4,8 @@
 - [#515](https://github.com/wycats/handlebars.js/issues/515) - Add node require extensions support ([@jjclark1982](https://github.com/jjclark1982))
 - [#517](https://github.com/wycats/handlebars.js/issues/517) - Fix amd precompiler output with directories ([@blessenm](https://github.com/blessenm))
 - [#519](https://github.com/wycats/handlebars.js/issues/519) - Fix partials with . name ([@jamesgorrie](https://github.com/jamesgorrie))
+- [#433](https://github.com/wycats/handlebars.js/issues/433) - Add support for unicode ids
+- [#469](https://github.com/wycats/handlebars.js/issues/469) - Add support for `?` in ids
 - Docs updates
 
 [Commits](https://github.com/wycats/handlebars.js/compare/v1.0.11...master)
diff --git a/spec/parser_spec.rb b/spec/parser_spec.rb
@@ -125,6 +125,10 @@ def path(*parts)
 
   it "parses simple mustaches" do
     ast_for("{{foo}}").should == root { mustache id("foo") }
+    ast_for("{{foo?}}").should == root { mustache id("foo?") }
+    ast_for("{{foo_}}").should == root { mustache id("foo_") }
+    ast_for("{{foo-}}").should == root { mustache id("foo-") }
+    ast_for("{{foo:}}").should == root { mustache id("foo:") }
   end
 
   it "parses simple mustaches with data" do
diff --git a/src/handlebars.l b/src/handlebars.l
@@ -32,7 +32,7 @@
 <mu>"{{"                         { return 'OPEN'; }
 
 <mu>"="                          { return 'EQUALS'; }
-<mu>"."/[}/ ]                    { return 'ID'; }
+<mu>"."/[}\/ ]                   { return 'ID'; }
 <mu>".."                         { return 'ID'; }
 <mu>[\/.]                        { return 'SEP'; }
 <mu>\s+                          { /*ignore whitespace*/ }
@@ -44,7 +44,18 @@
 <mu>"true"/[}\s]                 { return 'BOOLEAN'; }
 <mu>"false"/[}\s]                { return 'BOOLEAN'; }
 <mu>\-?[0-9]+/[}\s]              { return 'INTEGER'; }
-<mu>[a-zA-Z0-9_$:\-]+/[=}\s\/.]  { return 'ID'; }
+
+/*
+ID is the inverse of control characters.
+Control characters ranges:
+  [\s]          Whitespace
+  [!"#%-,\./]   !, ", #, %, &, ', (, ), *, +, ,, ., /,  Exceptions in range: $, -
+  [;->@]        ;, <, =, >, @,                          Exceptions in range: :, ?
+  [\[-\^`]      [, \, ], ^, `,                          Exceptions in range: _
+  [\{-~]        {, |, }, ~
+*/
+<mu>[^\s!"#%-,\.\/;->@\[-\^`\{-~]+/[=}\s\/.]   { return 'ID'; }
+
 <mu>'['[^\]]*']'                 { yytext = yytext.substr(1, yyleng-2); return 'ID'; }
 <mu>.                            { return 'INVALID'; }
 <par>\s+                         { /*ignore whitespace*/ }

Original file line number	Diff line number	Diff line change
`@@ -644,7 +644,7 @@ case 33: return 5;`
`644`	`644`	`break;`
`645`	`645`	`}`
`646`	`646`	`};`
`647`		-lexer.rules = [/^(?:\\\\(?=(\{\{)))/,/^(?:[^\x00]?(?=(\{\{)))/,/^(?:[^\x00]+)/,/^(?:[^\x00]{2,}?(?=(\{\{\|$)))/,/^(?:[\s\S]?--\}\})/,/^(?:\{\{>)/,/^(?:\{\{#)/,/^(?:\{\{\/)/,/^(?:\{\{\^)/,/^(?:\{\{\selse\b)/,/^(?:\{\{\{)/,/^(?:\{\{&)/,/^(?:\{\{!--)/,/^(?:\{\{![\s\S]?\}\})/,/^(?:\{\{)/,/^(?:=)/,/^(?:\.(?=[}/ ]))/,/^(?:\.\.)/,/^(?:[\/.])/,/^(?:\s+)/,/^(?:\}\}\})/,/^(?:\}\})/,/^(?:"(\\["]\|[^"])")/,/^(?:'(\\[']\|[^'])')/,/^(?:@[a-zA-Z]+)/,/^(?:true(?=[}\s]))/,/^(?:false(?=[}\s]))/,/^(?:-?[0-9]+(?=[}\s]))/,/^(?:[a-zA-Z0-9_$:\-]+(?=[=}\s\/.]))/,/^(?:\[[^\]]*\])/,/^(?:.)/,/^(?:\s+)/,/^(?:[a-zA-Z0-9_$\-\/\.]+)/,/^(?:$)/];
	`647`	+lexer.rules = [/^(?:\\\\(?=(\{\{)))/,/^(?:[^\x00]?(?=(\{\{)))/,/^(?:[^\x00]+)/,/^(?:[^\x00]{2,}?(?=(\{\{\|$)))/,/^(?:[\s\S]?--\}\})/,/^(?:\{\{>)/,/^(?:\{\{#)/,/^(?:\{\{\/)/,/^(?:\{\{\^)/,/^(?:\{\{\selse\b)/,/^(?:\{\{\{)/,/^(?:\{\{&)/,/^(?:\{\{!--)/,/^(?:\{\{![\s\S]?\}\})/,/^(?:\{\{)/,/^(?:=)/,/^(?:\.(?=[}\/ ]))/,/^(?:\.\.)/,/^(?:[\/.])/,/^(?:\s+)/,/^(?:\}\}\})/,/^(?:\}\})/,/^(?:"(\\["]\|[^"])")/,/^(?:'(\\[']\|[^'])')/,/^(?:@[a-zA-Z]+)/,/^(?:true(?=[}\s]))/,/^(?:false(?=[}\s]))/,/^(?:-?[0-9]+(?=[}\s]))/,/^(?:[^\s!"#%-,\.\/;->@\[-\^`\{-~]+(?=[=}\s\/.]))/,/^(?:\[[^\]]*\])/,/^(?:.)/,/^(?:\s+)/,/^(?:[a-zA-Z0-9_$\-\/\.]+)/,/^(?:$)/];
`648`	`648`	`lexer.conditions = {"mu":{"rules":[5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,33],"inclusive":false},"emu":{"rules":[3],"inclusive":false},"com":{"rules":[4],"inclusive":false},"par":{"rules":[31,32],"inclusive":false},"INITIAL":{"rules":[0,1,2,33],"inclusive":true}};`
`649`	`649`	`return lexer;})()`
`650`	`650`	`parser.lexer = lexer;`