Upgrade to Postgres 16

.github/workflows/ci.yml

@@ -20,20 +20,20 @@ jobs:
  id: cache-protobuf
  uses: actions/cache@v1
  with:
- path: protobuf-3.14.0
- key: ${{ runner.os }}-protobuf-cpp-3.14.0
+ path: protobuf-25.1
+ key: ${{ runner.os }}-protobuf-25.1
  - name: Build protobuf library
  if: matrix.protobuf_lib == 'protobuf-cpp' && steps.cache-protobuf.outputs.cache-hit != 'true'
  run: |
- wget https:/protocolbuffers/protobuf/releases/download/v3.14.0/protobuf-cpp-3.14.0.tar.gz
- tar -xf protobuf-cpp-3.14.0.tar.gz
- cd protobuf-3.14.0
- ./configure
- make
+ git clone --depth 1 --branch v25.1 https:/protocolbuffers/protobuf.git protobuf-25.1
+ cd protobuf-25.1
+ git submodule update --init --recursive
+ cmake .
+ cmake --build . --parallel 10
  - name: Install protobuf library
  if: matrix.protobuf_lib == 'protobuf-cpp'
  run: |
- cd protobuf-3.14.0
+ cd protobuf-25.1
  sudo make install
  sudo ldconfig
  - name: Install Valgrind

Makefile

@@ -6,9 +6,9 @@ ARLIB = lib$(TARGET).a
 PGDIR = $(root_dir)/tmp/postgres
 PGDIRBZ2 = $(root_dir)/tmp/postgres.tar.bz2
 
-PG_VERSION = 15.1
+PG_VERSION = 16.1
 PG_VERSION_MAJOR = $(call word-dot,$(PG_VERSION),1)
-PROTOC_VERSION = 3.14.0
+PROTOC_VERSION = 25.1
 
 VERSION = 4.2.3
 VERSION_MAJOR = $(call word-dot,$(VERSION),1)
@@ -30,12 +30,12 @@ else
 endif
 
 SRC_FILES := $(wildcard src/*.c src/postgres/*.c) vendor/protobuf-c/protobuf-c.c vendor/xxhash/xxhash.c protobuf/pg_query.pb-c.c
-NOT_OBJ_FILES := src/pg_query_enum_defs.o src/pg_query_fingerprint_defs.o src/pg_query_fingerprint_conds.o src/pg_query_outfuncs_defs.o src/pg_query_outfuncs_conds.o src/pg_query_readfuncs_defs.o src/pg_query_readfuncs_conds.o src/postgres/guc-file.o src/postgres/scan.o src/pg_query_json_helper.o
+NOT_OBJ_FILES := src/pg_query_enum_defs.o src/pg_query_fingerprint_defs.o src/pg_query_fingerprint_conds.o src/pg_query_outfuncs_defs.o src/pg_query_outfuncs_conds.o src/pg_query_readfuncs_defs.o src/pg_query_readfuncs_conds.o src/postgres/guc-file.o src/postgres/scan.o src/pg_query_json_helper.o $(patsubst %.c,%.o,$(wildcard src/postgres/src_backend_nodes_*.funcs.c))
 OBJ_FILES := $(filter-out $(NOT_OBJ_FILES), $(SRC_FILES:.c=.o))
 
 override CFLAGS += -g -I. -I./vendor -I./src/postgres/include -Wall -Wno-unused-function -Wno-unused-value -Wno-unused-variable -fno-strict-aliasing -fwrapv -fPIC
 
-override PG_CONFIGURE_FLAGS += -q --without-readline --without-zlib
+override PG_CONFIGURE_FLAGS += -q --without-readline --without-zlib --without-icu
 
 override TEST_CFLAGS += -g -I. -I./vendor -Wall
 override TEST_LDFLAGS += -pthread
@@ -80,13 +80,13 @@ CC ?= cc
 # Experimental use of Protobuf C++ library, primarily used to validate JSON output matches Protobuf JSON mapping
 CXX_SRC_FILES := src/pg_query_outfuncs_protobuf_cpp.cc protobuf/pg_query.pb.cc
 ifeq ($(USE_PROTOBUF_CPP),1)
- override CXXFLAGS += `pkg-config --cflags protobuf` -I. -I./src/postgres/include -DHAVE_PTHREAD -std=c++11 -Wall -Wno-unused-function -Wno-zero-length-array -Wno-c99-extensions -fwrapv -fPIC
+ override CXXFLAGS += `pkg-config --cflags protobuf` -I. -I./src/postgres/include -DHAVE_PTHREAD -std=c++17 -Wall -Wno-unused-function -Wno-zero-length-array -Wno-c99-extensions -fwrapv -fPIC
  ifeq ($(DEBUG),1)
  override CXXFLAGS += -O0 -g
  else
  override CXXFLAGS += -O3 -g
  endif
- override TEST_LDFLAGS += `pkg-config --libs protobuf` -lstdc++
+ override TEST_LDFLAGS += `pkg-config --libs protobuf` -lstdc++ -lm
 
  # Don't use regular Protobuf-C or JSON implementation (instead implement the same methods using the C++ library)
  SRC_FILES := $(filter-out src/pg_query_outfuncs_json.c src/pg_query_outfuncs_protobuf.c, $(SRC_FILES))
@@ -120,7 +120,7 @@ $(PGDIR):
  cd $(PGDIR); patch -p1 < $(root_dir)/patches/06_alloc_set_delete_free_list.patch
  cd $(PGDIR); patch -p1 < $(root_dir)/patches/07_plpgsql_start_finish_datums.patch
  cd $(PGDIR); patch -p1 < $(root_dir)/patches/08_avoid_zero_length_delimiter_in_regression_tests.patch
- cd $(PGDIR); patch -p1 < $(root_dir)/patches/09_allow_trailing_junk.patch
+ cd $(PGDIR); patch -p1 < $(root_dir)/patches/09_allow_param_junk.patch
  cd $(PGDIR); ./configure $(PG_CONFIGURE_FLAGS)
  cd $(PGDIR); rm src/pl/plpgsql/src/pl_gram.h
  cd $(PGDIR); make -C src/pl/plpgsql/src pl_gram.h

patches/01_parser_additional_param_ref_support.patch

@@ -23,10 +23,10 @@ Date: Sun Jan 3 15:06:25 2021 -0800
  SELECT INTERVAL (6) $1;
 
 diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
-index 74339aa4db..4529fea74e 100644
+index c31b373358..164b9586c1 100644
 --- a/src/backend/parser/gram.y
 +++ b/src/backend/parser/gram.y
-@@ -168,6 +168,8 @@ static Node *makeBitStringConst(char *str, int location);
+@@ -178,6 +178,8 @@ static Node *makeBoolAConst(bool state, int location);
  static Node *makeBitStringConst(char *str, int location);
  static Node *makeNullAConst(int location);
  static Node *makeAConst(Node *v, int location);
@@ -35,9 +35,9 @@ index 74339aa4db..4529fea74e 100644
  static RoleSpec *makeRoleSpec(RoleSpecType type, int location);
  static void check_qualified_name(List *names, core_yyscan_t yyscanner);
  static List *check_func_name(List *names, core_yyscan_t yyscanner);
-@@ -1086,6 +1088,11 @@ AlterOptRoleElem:
+@@ -1176,6 +1178,11 @@ AlterOptRoleElem:
  $$ = makeDefElem("password",
- (Node *)makeString($2), @1);
+ (Node *) makeString($2), @1);
  }
 + | PASSWORD PARAM
 + {
@@ -47,9 +47,9 @@ index 74339aa4db..4529fea74e 100644
  | PASSWORD NULL_P
  {
  $$ = makeDefElem("password", NULL, @1);
-@@ -1100,6 +1107,16 @@ AlterOptRoleElem:
+@@ -1190,6 +1197,16 @@ AlterOptRoleElem:
  $$ = makeDefElem("password",
- (Node *)makeString($3), @1);
+ (Node *) makeString($3), @1);
  }
 + | ENCRYPTED PASSWORD PARAM
 + {
@@ -64,7 +64,7 @@ index 74339aa4db..4529fea74e 100644
  | UNENCRYPTED PASSWORD Sconst
  {
  ereport(ERROR,
-@@ -1587,6 +1604,14 @@ set_rest_more: /* Generic SET syntaxes: */
+@@ -1703,6 +1720,14 @@ set_rest_more: /* Generic SET syntaxes: */
  n->args = list_make1(makeStringConst($2, @2));
  $$ = n;
  }
@@ -79,7 +79,7 @@ index 74339aa4db..4529fea74e 100644
  | NAMES opt_encoding
  {
  VariableSetStmt *n = makeNode(VariableSetStmt);
-@@ -1606,6 +1631,14 @@ set_rest_more: /* Generic SET syntaxes: */
+@@ -1724,6 +1749,14 @@ set_rest_more: /* Generic SET syntaxes: */
  n->args = list_make1(makeStringConst($2, @2));
  $$ = n;
  }
@@ -94,7 +94,7 @@ index 74339aa4db..4529fea74e 100644
  | SESSION AUTHORIZATION NonReservedWord_or_Sconst
  {
  VariableSetStmt *n = makeNode(VariableSetStmt);
-@@ -1614,6 +1647,14 @@ set_rest_more: /* Generic SET syntaxes: */
+@@ -1733,6 +1766,14 @@ set_rest_more: /* Generic SET syntaxes: */
  n->args = list_make1(makeStringConst($3, @3));
  $$ = n;
  }
@@ -109,7 +109,7 @@ index 74339aa4db..4529fea74e 100644
  | SESSION AUTHORIZATION DEFAULT
  {
  VariableSetStmt *n = makeNode(VariableSetStmt);
-@@ -1653,6 +1694,8 @@ var_value: opt_boolean_or_string
+@@ -1775,6 +1816,8 @@ var_value: opt_boolean_or_string
  { $$ = makeStringConst($1, @1); }
  | NumericOnly
  { $$ = makeAConst($1, @1); }
@@ -118,7 +118,7 @@ index 74339aa4db..4529fea74e 100644
  ;
 
  iso_level: READ UNCOMMITTED { $$ = "read uncommitted"; }
-@@ -1686,6 +1729,10 @@ zone_value:
+@@ -1808,6 +1851,10 @@ zone_value:
  {
  $$ = makeStringConst($1, @1);
  }
@@ -129,7 +129,7 @@ index 74339aa4db..4529fea74e 100644
  | IDENT
  {
  $$ = makeStringConst($1, @1);
-@@ -14783,6 +14830,10 @@ extract_list:
+@@ -16125,6 +16172,10 @@ extract_list:
  {
  $$ = list_make2(makeStringConst($1, @1), $3);
  }
@@ -138,9 +138,9 @@ index 74339aa4db..4529fea74e 100644
 + $$ = list_make2(makeParamRef($1, @1), $3);
 + }
  ;
-
+ 
  /* Allow delimited string Sconst in extract_arg as an SQL extension.
-@@ -15205,6 +15256,45 @@ AexprConst: Iconst
+@@ -16669,6 +16720,45 @@ AexprConst: Iconst
  t->location = @1;
  $$ = makeStringConstCast($6, @6, t);
  }
@@ -186,7 +186,7 @@ index 74339aa4db..4529fea74e 100644
  | ConstTypename Sconst
  {
  $$ = makeStringConstCast($2, @2, $1);
-@@ -15222,6 +15312,23 @@ AexprConst: Iconst
+@@ -16688,6 +16778,23 @@ AexprConst: Iconst
  makeIntConst($3, @3));
  $$ = makeStringConstCast($5, @5, t);
  }
@@ -210,8 +210,8 @@ index 74339aa4db..4529fea74e 100644
  | TRUE_P
  {
  $$ = makeBoolAConst(true, @1);
-@@ -16599,6 +1676,24 @@ makeBoolAConst(bool state, int location)
- return makeTypeCast((Node *)n, SystemTypeName("bool"), -1);
+@@ -18093,6 +18200,24 @@ makeAConst(Node *v, int location)
+ return n;
  }
 
 +/* makeParamRef

patches/03_lexer_track_yyllocend.patch

@@ -8,18 +8,18 @@ Date: Sun Jan 3 15:59:40 2021 -0800
  as this is made available by pg_query for uses such as syntax highlighting.
 
 diff --git a/src/backend/parser/scan.l b/src/backend/parser/scan.l
-index a0dfcf3b99..768170bb30 100644
+index b2216a9eac..a62798cf98 100644
 --- a/src/backend/parser/scan.l
 +++ b/src/backend/parser/scan.l
-@@ -513,6 +513,7 @@ other .
+@@ -537,6 +537,7 @@ other .
  {
  /* If NCHAR isn't a keyword, just return "n" */
  yylval->str = pstrdup("n");
 + yyextra->yyllocend = yytext - yyextra->scanbuf + yyleng;
  return IDENT;
  }
  }
-@@ -581,9 +582,11 @@ other .
+@@ -605,9 +606,11 @@ other .
  {
  case xb:
  yylval->str = litbufdup(yyscanner);
@@ -31,7 +31,7 @@ index a0dfcf3b99..768170bb30 100644
  return XCONST;
  case xq:
  case xe:
-@@ -596,9 +599,11 @@ other .
+@@ -620,9 +623,11 @@ other .
  yyextra->literallen,
  false);
  yylval->str = litbufdup(yyscanner);
@@ -43,39 +43,39 @@ index a0dfcf3b99..768170bb30 100644
  return USCONST;
  default:
  yyerror("unhandled previous state in xqs");
-@@ -736,6 +741,7 @@ other .
+@@ -760,6 +765,7 @@ other .
  yyextra->dolqstart = NULL;
  BEGIN(INITIAL);
  yylval->str = litbufdup(yyscanner);
 + yyextra->yyllocend = yytext - yyextra->scanbuf + yyleng;
  return SCONST;
  }
  else
-@@ -781,6 +787,7 @@ other .
+@@ -805,6 +811,7 @@ other .
  if (yyextra->literallen >= NAMEDATALEN)
  truncate_identifier(ident, yyextra->literallen, true);
  yylval->str = ident;
 + yyextra->yyllocend = yytext - yyextra->scanbuf + yyleng;
  return IDENT;
  }
  <xui>{dquote} {
-@@ -789,6 +796,7 @@ other .
+@@ -813,6 +820,7 @@ other .
  yyerror("zero-length delimited identifier");
  /* can't truncate till after we de-escape the ident */
  yylval->str = litbufdup(yyscanner);
 + yyextra->yyllocend = yytext - yyextra->scanbuf + yyleng;
  return UIDENT;
  }
  <xd,xui>{xddouble} {
-@@ -808,6 +816,7 @@ other .
+@@ -832,6 +840,7 @@ other .
  /* and treat it as {identifier} */
  ident = downcase_truncate_identifier(yytext, yyleng, true);
  yylval->str = ident;
 + yyextra->yyllocend = yytext - yyextra->scanbuf + yyleng;
  return IDENT;
  }
 
-@@ -1068,6 +1077,7 @@ other .
+@@ -1096,6 +1105,7 @@ other .
  */
  ident = downcase_truncate_identifier(yytext, yyleng, true);
  yylval->str = ident;
@@ -84,7 +84,7 @@ index a0dfcf3b99..768170bb30 100644
  }
 
 diff --git a/src/include/parser/scanner.h b/src/include/parser/scanner.h
-index a27352afc1..29852fbd86 100644
+index da013837cd..e2a73201c4 100644
 --- a/src/include/parser/scanner.h
 +++ b/src/include/parser/scanner.h
 @@ -113,6 +113,8 @@ typedef struct core_yy_extra_type

patches/04_lexer_comments_as_tokens.patch

@@ -12,10 +12,10 @@ Date: Sun Jan 3 16:01:44 2021 -0800
  this change, the lexer returns them as SQL_COMMENT/C_COMMENT tokens.
 
 diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
-index 8beb6fef5e..f2c963e801 100644
+index 164b9586c1..7dd5505cd1 100644
 --- a/src/backend/parser/gram.y
 +++ b/src/backend/parser/gram.y
-@@ -629,6 +629,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
+@@ -678,6 +678,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
  %token <ival> ICONST PARAM
  %token TYPECAST DOT_DOT COLON_EQUALS EQUALS_GREATER
  %token LESS_EQUALS GREATER_EQUALS NOT_EQUALS
@@ -24,12 +24,12 @@ index 8beb6fef5e..f2c963e801 100644
  /*
  * If you want to make any keyword changes, update the keyword table in
 diff --git a/src/backend/parser/parser.c b/src/backend/parser/parser.c
-index be86eb37fe..07b8fd486f 100644
+index e17c310cc1..9b2926878a 100644
 --- a/src/backend/parser/parser.c
 +++ b/src/backend/parser/parser.c
-@@ -150,6 +150,9 @@ base_yylex(YYSTYPE *lvalp, YYLTYPE *llocp, core_yyscan_t yyscanner)
- case USCONST:
- cur_token_length = strlen(yyextra->core_yy_extra.scanbuf + *llocp);
+@@ -156,6 +156,9 @@ base_yylex(YYSTYPE *lvalp, YYLTYPE *llocp, core_yyscan_t yyscanner)
+ case WITHOUT:
+ cur_token_length = 7;
  break;
 + case SQL_COMMENT:
 + case C_COMMENT:
@@ -38,10 +38,10 @@ index be86eb37fe..07b8fd486f 100644
  return cur_token;
  }
 diff --git a/src/backend/parser/scan.l b/src/backend/parser/scan.l
-index 768170bb30..24fd149497 100644
+index a62798cf98..154defc6b8 100644
 --- a/src/backend/parser/scan.l
 +++ b/src/backend/parser/scan.l
-@@ -223,7 +223,7 @@ non_newline [^\n\r]
+@@ -226,7 +226,7 @@ non_newline [^\n\r]
 
  comment ("--"{non_newline}*)
 
@@ -50,7 +50,7 @@ index 768170bb30..24fd149497 100644
 
  /*
  * SQL requires at least one newline in the whitespace separating
-@@ -232,8 +232,8 @@ whitespace ({space}+|{comment})
+@@ -235,8 +235,8 @@ whitespace ({space}+|{comment})
  * it, whereas {whitespace} should generally have a * after it...
  */
 
@@ -61,7 +61,7 @@ index 768170bb30..24fd149497 100644
  whitespace_with_newline ({horiz_whitespace}*{newline}{special_whitespace}*)
 
  quote '
-@@ -419,6 +419,11 @@ other .
+@@ -443,6 +443,11 @@ other .
  /* ignore */
  }
 
@@ -73,7 +73,7 @@ index 768170bb30..24fd149497 100644
  {xcstart} {
  /* Set location in case of syntax error in comment */
  SET_YYLLOC();
-@@ -437,7 +442,11 @@ other .
+@@ -461,7 +466,11 @@ other .
 
  {xcstop} {
  if (yyextra->xcdepth <= 0)
@@ -86,10 +86,10 @@ index 768170bb30..24fd149497 100644
  (yyextra->xcdepth)--;
  }
 diff --git a/src/interfaces/ecpg/preproc/parser.c b/src/interfaces/ecpg/preproc/parser.c
-index a2eeeba217..cbfb92384a 100644
+index 38e7acb680..4814158aaf 100644
 --- a/src/interfaces/ecpg/preproc/parser.c
 +++ b/src/interfaces/ecpg/preproc/parser.c
-@@ -84,6 +84,9 @@ filtered_base_yylex(void)
+@@ -86,6 +86,9 @@ filtered_base_yylex(void)
  case UIDENT:
  case USCONST:
  break;
@@ -100,10 +100,10 @@ index a2eeeba217..cbfb92384a 100644
  return cur_token;
  }
 diff --git a/src/pl/plpgsql/src/pl_gram.y b/src/pl/plpgsql/src/pl_gram.y
-index fc6cedb7a4..1cf9405d81 100644
+index 6a09bfdd67..fb1b519b27 100644
 --- a/src/pl/plpgsql/src/pl_gram.y
 +++ b/src/pl/plpgsql/src/pl_gram.y
-@@ -234,6 +234,7 @@ static void check_raise_parameters(PLpgSQL_stmt_raise *stmt);
+@@ -232,6 +232,7 @@ static void check_raise_parameters(PLpgSQL_stmt_raise *stmt);
  %token <ival> ICONST PARAM
  %token TYPECAST DOT_DOT COLON_EQUALS EQUALS_GREATER
  %token LESS_EQUALS GREATER_EQUALS NOT_EQUALS
@@ -112,7 +112,7 @@ index fc6cedb7a4..1cf9405d81 100644
  /*
  * Other tokens recognized by plpgsql's lexer interface layer (pl_scanner.c).
 diff --git a/src/pl/plpgsql/src/pl_scanner.c b/src/pl/plpgsql/src/pl_scanner.c
-index 9cea2e42ac..a0b86ac587 100644
+index 4e98af2395..40a17caaf9 100644
 --- a/src/pl/plpgsql/src/pl_scanner.c
 +++ b/src/pl/plpgsql/src/pl_scanner.c
 @@ -342,6 +342,11 @@ internal_yylex(TokenAuxData *auxdata)