Polish the tokenizer

git-svn-id: svn+ssh://projects.linpro.no/svn/varnish/trunk@2951 d4fa192b-c00b-0410-8231-f00ffab90ce4

diff --git a/varnish-cache/lib/libvcl/vcc_fixed_token.c b/varnish-cache/lib/libvcl/vcc_fixed_token.c
index 727467956f2e587d68d59cc697ea0858f6ec4ab2..4f5654c99b560a8d24672f5c8efd409332094893 100644 (file)
--- a/varnish-cache/lib/libvcl/vcc_fixed_token.c
+++ b/varnish-cache/lib/libvcl/vcc_fixed_token.c
@@ -18,205 +18,148 @@ vcl_fixed_token(const char *p, const char **q)
 
        switch (p[0]) {
        case '!':
-               if (p[0] == '!' && p[1] == '=') {
+               if (p[1] == '=') {
                        *q = p + 2;
                        return (T_NEQ);
                }
-               if (p[0] == '!') {
-                       *q = p + 1;
-                       return ('!');
-               }
-               return (0);
+               *q = p + 1;
+               return (p[0]);
        case '%':
-               if (p[0] == '%') {
-                       *q = p + 1;
-                       return ('%');
-               }
-               return (0);
+               *q = p + 1;
+               return (p[0]);
        case '&':
-               if (p[0] == '&' && p[1] == '&') {
+               if (p[1] == '&') {
                        *q = p + 2;
                        return (T_CAND);
                }
-               if (p[0] == '&') {
-                       *q = p + 1;
-                       return ('&');
-               }
-               return (0);
+               *q = p + 1;
+               return (p[0]);
        case '(':
-               if (p[0] == '(') {
-                       *q = p + 1;
-                       return ('(');
-               }
-               return (0);
+               *q = p + 1;
+               return (p[0]);
        case ')':
-               if (p[0] == ')') {
-                       *q = p + 1;
-                       return (')');
-               }
-               return (0);
+               *q = p + 1;
+               return (p[0]);
        case '*':
-               if (p[0] == '*' && p[1] == '=') {
+               if (p[1] == '=') {
                        *q = p + 2;
                        return (T_MUL);
                }
-               if (p[0] == '*') {
-                       *q = p + 1;
-                       return ('*');
-               }
-               return (0);
+               *q = p + 1;
+               return (p[0]);
        case '+':
-               if (p[0] == '+' && p[1] == '=') {
+               if (p[1] == '=') {
                        *q = p + 2;
                        return (T_INCR);
                }
-               if (p[0] == '+' && p[1] == '+') {
+               if (p[1] == '+') {
                        *q = p + 2;
                        return (T_INC);
                }
-               if (p[0] == '+') {
-                       *q = p + 1;
-                       return ('+');
-               }
-               return (0);
+               *q = p + 1;
+               return (p[0]);
        case ',':
-               if (p[0] == ',') {
-                       *q = p + 1;
-                       return (',');
-               }
-               return (0);
+               *q = p + 1;
+               return (p[0]);
        case '-':
-               if (p[0] == '-' && p[1] == '=') {
+               if (p[1] == '=') {
                        *q = p + 2;
                        return (T_DECR);
                }
-               if (p[0] == '-' && p[1] == '-') {
+               if (p[1] == '-') {
                        *q = p + 2;
                        return (T_DEC);
                }
-               if (p[0] == '-') {
-                       *q = p + 1;
-                       return ('-');
-               }
-               return (0);
+               *q = p + 1;
+               return (p[0]);
        case '.':
-               if (p[0] == '.') {
-                       *q = p + 1;
-                       return ('.');
-               }
-               return (0);
+               *q = p + 1;
+               return (p[0]);
        case '/':
-               if (p[0] == '/' && p[1] == '=') {
+               if (p[1] == '=') {
                        *q = p + 2;
                        return (T_DIV);
                }
-               if (p[0] == '/') {
-                       *q = p + 1;
-                       return ('/');
-               }
-               return (0);
+               *q = p + 1;
+               return (p[0]);
        case ';':
-               if (p[0] == ';') {
-                       *q = p + 1;
-                       return (';');
-               }
-               return (0);
+               *q = p + 1;
+               return (p[0]);
        case '<':
-               if (p[0] == '<' && p[1] == '=') {
+               if (p[1] == '=') {
                        *q = p + 2;
                        return (T_LEQ);
                }
-               if (p[0] == '<' && p[1] == '<') {
+               if (p[1] == '<') {
                        *q = p + 2;
                        return (T_SHL);
                }
-               if (p[0] == '<') {
-                       *q = p + 1;
-                       return ('<');
-               }
-               return (0);
+               *q = p + 1;
+               return (p[0]);
        case '=':
-               if (p[0] == '=' && p[1] == '=') {
+               if (p[1] == '=') {
                        *q = p + 2;
                        return (T_EQ);
                }
-               if (p[0] == '=') {
-                       *q = p + 1;
-                       return ('=');
-               }
-               return (0);
+               *q = p + 1;
+               return (p[0]);
        case '>':
-               if (p[0] == '>' && p[1] == '>') {
+               if (p[1] == '>') {
                        *q = p + 2;
                        return (T_SHR);
                }
-               if (p[0] == '>' && p[1] == '=') {
+               if (p[1] == '=') {
                        *q = p + 2;
                        return (T_GEQ);
                }
-               if (p[0] == '>') {
-                       *q = p + 1;
-                       return ('>');
-               }
-               return (0);
+               *q = p + 1;
+               return (p[0]);
        case 'e':
-               if (p[0] == 'e' && p[1] == 'l' && p[2] == 's' && 
+               if (p[1] == 'l' && p[2] == 's' && 
                    p[3] == 'i' && p[4] == 'f' && !isvar(p[5])) {
                        *q = p + 5;
                        return (T_ELSIF);
                }
-               if (p[0] == 'e' && p[1] == 'l' && p[2] == 's' && 
+               if (p[1] == 'l' && p[2] == 's' && 
                    p[3] == 'e' && p[4] == 'i' && p[5] == 'f'
                     && !isvar(p[6])) {
                        *q = p + 6;
                        return (T_ELSEIF);
                }
-               if (p[0] == 'e' && p[1] == 'l' && p[2] == 's' && 
+               if (p[1] == 'l' && p[2] == 's' && 
                    p[3] == 'e' && !isvar(p[4])) {
                        *q = p + 4;
                        return (T_ELSE);
                }
                return (0);
        case 'i':
-               if (p[0] == 'i' && p[1] == 'n' && p[2] == 'c' && 
+               if (p[1] == 'n' && p[2] == 'c' && 
                    p[3] == 'l' && p[4] == 'u' && p[5] == 'd' && 
                    p[6] == 'e' && !isvar(p[7])) {
                        *q = p + 7;
                        return (T_INCLUDE);
                }
-               if (p[0] == 'i' && p[1] == 'f' && !isvar(p[2])) {
+               if (p[1] == 'f' && !isvar(p[2])) {
                        *q = p + 2;
                        return (T_IF);
                }
                return (0);
        case '{':
-               if (p[0] == '{') {
-                       *q = p + 1;
-                       return ('{');
-               }
-               return (0);
+               *q = p + 1;
+               return (p[0]);
        case '|':
-               if (p[0] == '|' && p[1] == '|') {
+               if (p[1] == '|') {
                        *q = p + 2;
                        return (T_COR);
                }
-               if (p[0] == '|') {
-                       *q = p + 1;
-                       return ('|');
-               }
-               return (0);
+               *q = p + 1;
+               return (p[0]);
        case '}':
-               if (p[0] == '}') {
-                       *q = p + 1;
-                       return ('}');
-               }
-               return (0);
+               *q = p + 1;
+               return (p[0]);
        case '~':
-               if (p[0] == '~') {
-                       *q = p + 1;
-                       return ('~');
-               }
-               return (0);
+               *q = p + 1;
+               return (p[0]);
        default:
                return (0);
        }

diff --git a/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl b/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl
index 2362f2c365df02cb6adadf9e9aa2411cea2c8b51..1c23bb7a304dabfb2dd04c74554c19d55cd279c5 100755 (executable)
--- a/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl
+++ b/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl
@@ -260,8 +260,6 @@ foreach t $fixed {
 }
 set seq [lsort [array names xx]]
 
-set ll 0
-
 puts $fo {
 unsigned
 vcl_fixed_token(const char *p, const char **q)}
@@ -280,19 +278,18 @@ foreach ch "$seq" {
        # And do then in reverse order to match longest first
        set l [lsort -index 0 -decreasing $l]
        scan "$ch" "%c" cx
-       if {$cx != $ll} {
-               if {$ll} {
-                       puts $fo "              return (0);"
-               }
-       
-               puts $fo "      case '$ch':"
-               set ll $cx
-       }
+       puts $fo "      case '$ch':"
+       set retval "0"
        foreach tt $l {
                set k [lindex $tt 0]
+               if {[string length $k] == 1} {
+                       puts $fo "\t\t*q = p + 1;"
+                       set retval {p[0]}
+                       continue;
+               }
                puts -nonewline $fo "           if ("
-               for {set i 0} {$i < [string length $k]} {incr i} {
-                       if {$i > 0} {
+               for {set i 1} {$i < [string length $k]} {incr i} {
+                       if {$i > 1} {
                                puts -nonewline $fo " && "
                                if {![expr $i % 3]} {
                                        puts -nonewline $fo "\n\t\t    "
@@ -307,12 +304,13 @@ foreach ch "$seq" {
                        puts -nonewline $fo " && !isvar(p\[$i\])"
                }
                puts $fo ") {"
-               puts $fo "                      *q = p + [string length $k];"
-               puts $fo "                      return ([lindex $tt 1]);"
-               puts $fo "              }"
+               puts $fo "\t\t\t*q = p + [string length $k];"
+               puts $fo "\t\t\treturn ([lindex $tt 1]);"
+               puts $fo "\t\t}"
        }
+       puts $fo "\t\treturn ($retval);"
 } 
-puts $fo "             return (0);"
+
 puts $fo "     default:"
 puts $fo "             return (0);"
 puts $fo "     }"