From 2a97f787929415f990098266a9197e1fe4f23b5f Mon Sep 17 00:00:00 2001
From: phk <phk@d4fa192b-c00b-0410-8231-f00ffab90ce4>
Date: Fri, 18 Jul 2008 21:30:01 +0000
Subject: [PATCH] Polish the tokenizer

git-svn-id: svn+ssh://projects.linpro.no/svn/varnish/trunk@2951 d4fa192b-c00b-0410-8231-f00ffab90ce4
---
 varnish-cache/lib/libvcl/vcc_fixed_token.c    | 171 ++++++------------
 .../lib/libvcl/vcc_gen_fixed_token.tcl        |  30 ++-
 2 files changed, 71 insertions(+), 130 deletions(-)

diff --git a/varnish-cache/lib/libvcl/vcc_fixed_token.c b/varnish-cache/lib/libvcl/vcc_fixed_token.c
index 72746795..4f5654c9 100644
--- a/varnish-cache/lib/libvcl/vcc_fixed_token.c
+++ b/varnish-cache/lib/libvcl/vcc_fixed_token.c
@@ -18,205 +18,148 @@ vcl_fixed_token(const char *p, const char **q)
 
 	switch (p[0]) {
 	case '!':
-		if (p[0] == '!' && p[1] == '=') {
+		if (p[1] == '=') {
 			*q = p + 2;
 			return (T_NEQ);
 		}
-		if (p[0] == '!') {
-			*q = p + 1;
-			return ('!');
-		}
-		return (0);
+		*q = p + 1;
+		return (p[0]);
 	case '%':
-		if (p[0] == '%') {
-			*q = p + 1;
-			return ('%');
-		}
-		return (0);
+		*q = p + 1;
+		return (p[0]);
 	case '&':
-		if (p[0] == '&' && p[1] == '&') {
+		if (p[1] == '&') {
 			*q = p + 2;
 			return (T_CAND);
 		}
-		if (p[0] == '&') {
-			*q = p + 1;
-			return ('&');
-		}
-		return (0);
+		*q = p + 1;
+		return (p[0]);
 	case '(':
-		if (p[0] == '(') {
-			*q = p + 1;
-			return ('(');
-		}
-		return (0);
+		*q = p + 1;
+		return (p[0]);
 	case ')':
-		if (p[0] == ')') {
-			*q = p + 1;
-			return (')');
-		}
-		return (0);
+		*q = p + 1;
+		return (p[0]);
 	case '*':
-		if (p[0] == '*' && p[1] == '=') {
+		if (p[1] == '=') {
 			*q = p + 2;
 			return (T_MUL);
 		}
-		if (p[0] == '*') {
-			*q = p + 1;
-			return ('*');
-		}
-		return (0);
+		*q = p + 1;
+		return (p[0]);
 	case '+':
-		if (p[0] == '+' && p[1] == '=') {
+		if (p[1] == '=') {
 			*q = p + 2;
 			return (T_INCR);
 		}
-		if (p[0] == '+' && p[1] == '+') {
+		if (p[1] == '+') {
 			*q = p + 2;
 			return (T_INC);
 		}
-		if (p[0] == '+') {
-			*q = p + 1;
-			return ('+');
-		}
-		return (0);
+		*q = p + 1;
+		return (p[0]);
 	case ',':
-		if (p[0] == ',') {
-			*q = p + 1;
-			return (',');
-		}
-		return (0);
+		*q = p + 1;
+		return (p[0]);
 	case '-':
-		if (p[0] == '-' && p[1] == '=') {
+		if (p[1] == '=') {
 			*q = p + 2;
 			return (T_DECR);
 		}
-		if (p[0] == '-' && p[1] == '-') {
+		if (p[1] == '-') {
 			*q = p + 2;
 			return (T_DEC);
 		}
-		if (p[0] == '-') {
-			*q = p + 1;
-			return ('-');
-		}
-		return (0);
+		*q = p + 1;
+		return (p[0]);
 	case '.':
-		if (p[0] == '.') {
-			*q = p + 1;
-			return ('.');
-		}
-		return (0);
+		*q = p + 1;
+		return (p[0]);
 	case '/':
-		if (p[0] == '/' && p[1] == '=') {
+		if (p[1] == '=') {
 			*q = p + 2;
 			return (T_DIV);
 		}
-		if (p[0] == '/') {
-			*q = p + 1;
-			return ('/');
-		}
-		return (0);
+		*q = p + 1;
+		return (p[0]);
 	case ';':
-		if (p[0] == ';') {
-			*q = p + 1;
-			return (';');
-		}
-		return (0);
+		*q = p + 1;
+		return (p[0]);
 	case '<':
-		if (p[0] == '<' && p[1] == '=') {
+		if (p[1] == '=') {
 			*q = p + 2;
 			return (T_LEQ);
 		}
-		if (p[0] == '<' && p[1] == '<') {
+		if (p[1] == '<') {
 			*q = p + 2;
 			return (T_SHL);
 		}
-		if (p[0] == '<') {
-			*q = p + 1;
-			return ('<');
-		}
-		return (0);
+		*q = p + 1;
+		return (p[0]);
 	case '=':
-		if (p[0] == '=' && p[1] == '=') {
+		if (p[1] == '=') {
 			*q = p + 2;
 			return (T_EQ);
 		}
-		if (p[0] == '=') {
-			*q = p + 1;
-			return ('=');
-		}
-		return (0);
+		*q = p + 1;
+		return (p[0]);
 	case '>':
-		if (p[0] == '>' && p[1] == '>') {
+		if (p[1] == '>') {
 			*q = p + 2;
 			return (T_SHR);
 		}
-		if (p[0] == '>' && p[1] == '=') {
+		if (p[1] == '=') {
 			*q = p + 2;
 			return (T_GEQ);
 		}
-		if (p[0] == '>') {
-			*q = p + 1;
-			return ('>');
-		}
-		return (0);
+		*q = p + 1;
+		return (p[0]);
 	case 'e':
-		if (p[0] == 'e' && p[1] == 'l' && p[2] == 's' && 
+		if (p[1] == 'l' && p[2] == 's' && 
 		    p[3] == 'i' && p[4] == 'f' && !isvar(p[5])) {
 			*q = p + 5;
 			return (T_ELSIF);
 		}
-		if (p[0] == 'e' && p[1] == 'l' && p[2] == 's' && 
+		if (p[1] == 'l' && p[2] == 's' && 
 		    p[3] == 'e' && p[4] == 'i' && p[5] == 'f'
 		     && !isvar(p[6])) {
 			*q = p + 6;
 			return (T_ELSEIF);
 		}
-		if (p[0] == 'e' && p[1] == 'l' && p[2] == 's' && 
+		if (p[1] == 'l' && p[2] == 's' && 
 		    p[3] == 'e' && !isvar(p[4])) {
 			*q = p + 4;
 			return (T_ELSE);
 		}
 		return (0);
 	case 'i':
-		if (p[0] == 'i' && p[1] == 'n' && p[2] == 'c' && 
+		if (p[1] == 'n' && p[2] == 'c' && 
 		    p[3] == 'l' && p[4] == 'u' && p[5] == 'd' && 
 		    p[6] == 'e' && !isvar(p[7])) {
 			*q = p + 7;
 			return (T_INCLUDE);
 		}
-		if (p[0] == 'i' && p[1] == 'f' && !isvar(p[2])) {
+		if (p[1] == 'f' && !isvar(p[2])) {
 			*q = p + 2;
 			return (T_IF);
 		}
 		return (0);
 	case '{':
-		if (p[0] == '{') {
-			*q = p + 1;
-			return ('{');
-		}
-		return (0);
+		*q = p + 1;
+		return (p[0]);
 	case '|':
-		if (p[0] == '|' && p[1] == '|') {
+		if (p[1] == '|') {
 			*q = p + 2;
 			return (T_COR);
 		}
-		if (p[0] == '|') {
-			*q = p + 1;
-			return ('|');
-		}
-		return (0);
+		*q = p + 1;
+		return (p[0]);
 	case '}':
-		if (p[0] == '}') {
-			*q = p + 1;
-			return ('}');
-		}
-		return (0);
+		*q = p + 1;
+		return (p[0]);
 	case '~':
-		if (p[0] == '~') {
-			*q = p + 1;
-			return ('~');
-		}
-		return (0);
+		*q = p + 1;
+		return (p[0]);
 	default:
 		return (0);
 	}
diff --git a/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl b/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl
index 2362f2c3..1c23bb7a 100755
--- a/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl
+++ b/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl
@@ -260,8 +260,6 @@ foreach t $fixed {
 }
 set seq [lsort [array names xx]]
 
-set ll 0
-
 puts $fo {
 unsigned
 vcl_fixed_token(const char *p, const char **q)}
@@ -280,19 +278,18 @@ foreach ch "$seq" {
 	# And do then in reverse order to match longest first
 	set l [lsort -index 0 -decreasing $l]
 	scan "$ch" "%c" cx
-	if {$cx != $ll} {
-		if {$ll} {
-			puts $fo "		return (0);"
-		}
-	
-		puts $fo "	case '$ch':"
-		set ll $cx
-	}
+	puts $fo "	case '$ch':"
+	set retval "0"
 	foreach tt $l {
 		set k [lindex $tt 0]
+		if {[string length $k] == 1} {
+			puts $fo "\t\t*q = p + 1;"
+			set retval {p[0]}
+			continue;
+		}
 		puts -nonewline $fo "		if ("
-		for {set i 0} {$i < [string length $k]} {incr i} {
-			if {$i > 0} {
+		for {set i 1} {$i < [string length $k]} {incr i} {
+			if {$i > 1} {
 				puts -nonewline $fo " && "
 				if {![expr $i % 3]} {
 					puts -nonewline $fo "\n\t\t    "
@@ -307,12 +304,13 @@ foreach ch "$seq" {
 			puts -nonewline $fo " && !isvar(p\[$i\])"
 		}
 		puts $fo ") {"
-		puts $fo "			*q = p + [string length $k];"
-		puts $fo "			return ([lindex $tt 1]);"
-		puts $fo "		}"
+		puts $fo "\t\t\t*q = p + [string length $k];"
+		puts $fo "\t\t\treturn ([lindex $tt 1]);"
+		puts $fo "\t\t}"
 	}
+	puts $fo "\t\treturn ($retval);"
 } 
-puts $fo "		return (0);"
+
 puts $fo "	default:"
 puts $fo "		return (0);"
 puts $fo "	}"
-- 
2.39.5