+sub tokenize {
+ my $line = shift || return;
+ my $line_ptr = $line;
+ my @line = ();
+
+ my $token_pattern = qr/[^"\s]+|"[^"]+"/;
+
+ while (my ($token) = $line_ptr =~ m/^($token_pattern)\s+/) {
+ $line_ptr = $';
+ push @line, $token;
+ }
+
+ if ($line_ptr =~ m/^$token_pattern$/) {
+ push @line, $line_ptr;
+ }
+ else {
+ my ($token) = split m/ /, $line_ptr, 1;
+ print STDERR "Failed to parse line: $line\n";
+ print STDERR "Parse error near token \"$token\".\n";
+ return;
+ }
+
+ foreach my $l (@line) {
+ if ($l =~ m/^"(.*)"$/) {
+ $l = $1;
+ }
+ }
+ return @line;
+}
+