Split psql's lexer into two separate .l files for SQL and backslash cases.

This gets us to a point where psqlscan.l can be used by other frontend
programs for the same purpose psql uses it for, ie to detect when it's
collected a complete SQL command from input that is divided across
line boundaries.  Moreover, other programs can supply their own lexers
for backslash commands of their own choosing.  A follow-on patch will
use this in pgbench.

The end result here is roughly the same as in Kyotaro Horiguchi's
0001-Make-SQL-parser-part-of-psqlscan-independent-from-ps.patch, although
the details of the method for switching between lexers are quite different.
Basically, in this patch we share the entire PsqlScanState, YY_BUFFER_STATE
stack, *and* yyscan_t between different lexers.  The only thing we need
to do to switch to a different lexer is to make sure the start_state is
valid for the new lexer.  This works because flex doesn't keep any other
persistent state that depends on the specific lexing tables generated for
a particular .l file.  (We are assuming that both lexers are built with
the same flex version, or at least versions that are compatible with
respect to the contents of yyscan_t; but that doesn't seem likely to
be a big problem in practice, considering how slowly flex changes.)

Aside from being more efficient than Horiguchi-san's original solution,
this avoids possible corner-case changes in semantics: the original code
was capable of popping the input buffer stack while still staying in
backslash-related parsing states.  I'm not sure that that equates to any
useful user-visible behaviors, but I'm not sure it doesn't either, so
I'm loath to assume that we only need to consider the topmost buffer when
parsing a backslash command.

I've attempted to update the MSVC build scripts for the added .l file,
but will rely on the buildfarm to see if I missed anything.

Kyotaro Horiguchi and Tom Lane

Author: tglsfdc

src/bin/psql/.gitignore

@@ -1,4 +1,5 @@
 /psqlscan.c
+/psqlscanslash.c
 /sql_help.h
 /sql_help.c
 /dumputils.c

src/bin/psql/Makefile

@@ -23,7 +23,7 @@ override CPPFLAGS := -I. -I$(srcdir) -I$(libpq_srcdir) -I$(top_srcdir)/src/bin/p
 OBJS=	command.o common.o help.o input.o stringutils.o mainloop.o copy.o \
 	startup.o prompt.o variables.o large_obj.o print.o describe.o \
 	tab-complete.o mbprint.o dumputils.o keywords.o kwlookup.o \
-	sql_help.o psqlscan.o \
+	sql_help.o psqlscan.o psqlscanslash.o \
 	$(WIN32RES)
 
 
@@ -47,12 +47,16 @@ sql_help.h: create_help.pl $(wildcard $(REFDOCDIR)/*.sgml)
 psqlscan.c: FLEXFLAGS = -Cfe -p -p
 psqlscan.c: FLEX_NO_BACKUP=yes
 
-# Latest flex causes warnings in this file.
+psqlscanslash.c: FLEXFLAGS = -Cfe -p -p
+psqlscanslash.c: FLEX_NO_BACKUP=yes
+
+# Latest flex causes warnings in these files.
 ifeq ($(GCC),yes)
 psqlscan.o: CFLAGS += -Wno-error
+psqlscanslash.o: CFLAGS += -Wno-error
 endif
 
-distprep: sql_help.h psqlscan.c
+distprep: sql_help.h psqlscan.c psqlscanslash.c
 
 install: all installdirs
 	$(INSTALL_PROGRAM) psql$(X) '$(DESTDIR)$(bindir)/psql$(X)'
@@ -64,9 +68,10 @@ installdirs:
 uninstall:
 	rm -f '$(DESTDIR)$(bindir)/psql$(X)' '$(DESTDIR)$(datadir)/psqlrc.sample'
 
-# psqlscan.c is in the distribution tarball, so is not cleaned here
 clean distclean:
 	rm -f psql$(X) $(OBJS) dumputils.c keywords.c kwlookup.c lex.backup
 
+# files removed here are supposed to be in the distribution tarball,
+# so do not clean them in the clean/distclean rules
 maintainer-clean: distclean
-	rm -f sql_help.h sql_help.c psqlscan.c
+	rm -f sql_help.h sql_help.c psqlscan.c psqlscanslash.c

src/bin/psql/command.c

@@ -45,7 +45,7 @@
 #include "large_obj.h"
 #include "mainloop.h"
 #include "print.h"
-#include "psqlscan.h"
+#include "psqlscanslash.h"
 #include "settings.h"
 #include "variables.h"
 

src/bin/psql/nls.mk

@@ -2,7 +2,8 @@
 CATALOG_NAME     = psql
 AVAIL_LANGUAGES  = cs de es fr it ja pl pt_BR ru zh_CN zh_TW
 GETTEXT_FILES    = command.c common.c copy.c help.c input.c large_obj.c \
-                   mainloop.c print.c psqlscan.c startup.c describe.c sql_help.h sql_help.c \
+                   mainloop.c print.c psqlscan.c psqlscanslash.c startup.c \
+                   describe.c sql_help.h sql_help.c \
                    tab-complete.c variables.c \
                    ../../common/exec.c ../../common/fe_memutils.c ../../common/username.c \
                    ../../common/wait_error.c

src/bin/psql/psqlscan.h

@@ -25,17 +25,6 @@ typedef enum
 	PSCAN_EOL					/* end of line, SQL possibly complete */
 } PsqlScanResult;
 
-/* Different ways for scan_slash_option to handle parameter words */
-enum slash_option_type
-{
-	OT_NORMAL,					/* normal case */
-	OT_SQLID,					/* treat as SQL identifier */
-	OT_SQLIDHACK,				/* SQL identifier, but don't downcase */
-	OT_FILEPIPE,				/* it's a filename or pipe */
-	OT_WHOLE_LINE,				/* just snarf the rest of the line */
-	OT_NO_EVAL					/* no expansion of backticks or variables */
-};
-
 /* Callback functions to be used by the lexer */
 typedef struct PsqlScanCallbacks
 {
@@ -61,15 +50,8 @@ extern PsqlScanResult psql_scan(PsqlScanState state,
 
 extern void psql_scan_reset(PsqlScanState state);
 
-extern bool psql_scan_in_quote(PsqlScanState state);
-
-extern char *psql_scan_slash_command(PsqlScanState state);
-
-extern char *psql_scan_slash_option(PsqlScanState state,
-					   enum slash_option_type type,
-					   char *quote,
-					   bool semicolon);
+extern void psql_scan_reselect_sql_lexer(PsqlScanState state);
 
-extern void psql_scan_slash_command_end(PsqlScanState state);
+extern bool psql_scan_in_quote(PsqlScanState state);
 
 #endif   /* PSQLSCAN_H */