From 259328d5eab4f7eedaa74ce57f8828da5e72c615 Mon Sep 17 00:00:00 2001 From: XL_up <3066352084@qq.com> Date: Mon, 28 Aug 2023 22:57:08 +0800 Subject: [PATCH 1/9] Update scansup.cpp --- src/common/backend/parser/scansup.cpp | 53 +++++++++++++++------------ 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/src/common/backend/parser/scansup.cpp b/src/common/backend/parser/scansup.cpp index dc11a225..3ee8ba1c 100644 --- a/src/common/backend/parser/scansup.cpp +++ b/src/common/backend/parser/scansup.cpp @@ -21,30 +21,29 @@ #include "parser/scansup.h" #include "mb/pg_wchar.h" -/* ---------------- - * scanstr - * - * if the string passed in has escaped codes, map the escape codes to actual - * chars +/* + * scanstr() --- if the string passed in has escaped codes, map the escape + * codes to actual chars. * * the string returned is palloc'd and should eventually be pfree'd by the * caller! - * ---------------- */ char* scanstr(const char* s) { char* newStr = NULL; int len, i, j; - if (s == NULL || s[0] == '\0') + if (s == NULL || s[0] == '\0') // The input string is empty. return pstrdup(""); len = strlen(s); - newStr = (char*)palloc(len + 1); /* string cannot get longer */ + // The new string constructed does not exceed the length of the original string. + // That is because the processes of the escape code would get the string shorter. + newStr = (char*)palloc(len + 1); /* string cannot get longer */ for (i = 0, j = 0; i < len; i++) { - if (s[i] == '\'') { + if (s[i] == '\'') { /* * Note: if scanner is working right, unescaped quotes can only * appear in pairs, so there should be another character. @@ -52,11 +51,11 @@ char* scanstr(const char* s) i++; newStr[j] = s[i]; } else if (s[i] == '\\') { - i++; + i++; // "\" and a next character are translated together into an escape symbol. switch (s[i]) { case 'b': newStr[j] = '\b'; - break; + break; case 'f': newStr[j] = '\f'; break; @@ -76,18 +75,20 @@ char* scanstr(const char* s) case '4': case '5': case '6': - case '7': { + case '7': { + // case '0'~'7', represents this is the highest bit of an octal number. int k; unsigned long octVal = 0; - + // Converts the next consecutive digits of up to three 0-7 digits into an octal number. for (k = 0; s[i + k] >= '0' && s[i + k] <= '7' && k < 3; k++){ octVal = (octVal << 3) + (s[i + k] - '0'); } i += k - 1; - newStr[j] = ((char)octVal); + // According to the octal numeric value corresponding to the ASCII code to determine the specific characters. + newStr[j] = ((char)octVal); } break; default: - newStr[j] = s[i]; + newStr[j] = s[i]; break; } /* switch */ } /* s[i] == '\\' */ @@ -96,7 +97,7 @@ char* scanstr(const char* s) } j++; } - newStr[j] = '\0'; + newStr[j] = '\0'; // The new string constructed should end with '\0'. return newStr; } @@ -120,6 +121,8 @@ char* downcase_truncate_identifier(const char* ident, int len, bool warn) bool enc_is_single_byte = false; result = (char*)palloc(len + 1); + // Gets the maximum byte of the database encoding. + // enc_is_single_byte = TRUE IF it is single-byte encoding. enc_is_single_byte = pg_database_encoding_max_length() == 1; /* @@ -131,7 +134,7 @@ char* downcase_truncate_identifier(const char* ident, int len, bool warn) * the high bit set, as long as they aren't part of a multi-byte character, * and use an ASCII-only downcasing for 7-bit characters. */ - for (i = 0; i < len; i++) { + for (i = 0; i < len; i++) { // Implementation of downcasing. unsigned char ch = (unsigned char)ident[i]; if (ch >= 'A' && ch <= 'Z') { @@ -143,7 +146,8 @@ char* downcase_truncate_identifier(const char* ident, int len, bool warn) } result[i] = '\0'; - if (i >= NAMEDATALEN) + // The length exceeds the identifier's maximum length, so truncate. + if (i >= NAMEDATALEN) truncate_identifier(result, i, warn); return result; @@ -152,15 +156,13 @@ char* downcase_truncate_identifier(const char* ident, int len, bool warn) /* * truncate_identifier() --- truncate an identifier to NAMEDATALEN-1 bytes. * - * The given string is modified in-place, if necessary. A warning is - * issued if requested. - * * We require the caller to pass in the string length since this saves a * strlen() call in some common usages. */ void truncate_identifier(char* ident, int len, bool warn) { if (len >= NAMEDATALEN) { + // Calculate the appropriate length after truncation. len = pg_mbcliplen(ident, len, NAMEDATALEN - 1); if (warn) { /* @@ -169,14 +171,17 @@ void truncate_identifier(char* ident, int len, bool warn) */ char buf[NAMEDATALEN]; errno_t rc; - + rc = memcpy_s(buf, NAMEDATALEN, ident, len); + securec_check(rc, "\0", "\0"); buf[len] = '\0'; + ereport(NOTICE, (errcode(ERRCODE_NAME_TOO_LONG), errmsg("identifier \"%s\" will be truncated to \"%s\"", ident, buf))); } - ident[len] = '\0'; + // Truncate by setting the end character of the identifier to '\0'. + ident[len] = '\0'; } } @@ -197,4 +202,4 @@ bool scanner_isspace(char ch) } return false; -} +} \ No newline at end of file -- 2.34.1 From 67fa6e09bd171476add64414b5cb31c31d37e1f6 Mon Sep 17 00:00:00 2001 From: XL_up <3066352084@qq.com> Date: Mon, 28 Aug 2023 22:58:20 +0800 Subject: [PATCH 2/9] Update keywords.cpp --- src/common/backend/parser/keywords.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/common/backend/parser/keywords.cpp b/src/common/backend/parser/keywords.cpp index 785635e2..c0184514 100644 --- a/src/common/backend/parser/keywords.cpp +++ b/src/common/backend/parser/keywords.cpp @@ -21,9 +21,11 @@ #define PG_KEYWORD(a, b, c) {a, b, c}, +// A table that stores keywords information. +// This table will be used in keyword-matching process. const ScanKeyword ScanKeywords[] = { #include "parser/kwlist.h" }; +// The number of table items, which is also the total number of keywords const int NumScanKeywords = lengthof(ScanKeywords); - -- 2.34.1 From f9faeac978fcaed82336d2ca3cf35e96e535da96 Mon Sep 17 00:00:00 2001 From: XL_up <3066352084@qq.com> Date: Mon, 28 Aug 2023 22:59:08 +0800 Subject: [PATCH 3/9] Update kwlookup.cpp --- src/common/backend/parser/kwlookup.cpp | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/common/backend/parser/kwlookup.cpp b/src/common/backend/parser/kwlookup.cpp index ba028bae..899c96a1 100644 --- a/src/common/backend/parser/kwlookup.cpp +++ b/src/common/backend/parser/kwlookup.cpp @@ -28,7 +28,7 @@ * * Returns a pointer to the ScanKeyword table entry, or NULL if no match. * - * The match is done case-insensitively. Note that we deliberately use a + * The match is done case-insensitively. Note that we deliberately use a * dumbed-down case conversion that will only translate 'A'-'Z' into 'a'-'z', * even if we are in a locale where tolower() would produce more or different * translations. This is to conform to the SQL99 spec, which says that @@ -36,13 +36,20 @@ * receive a different case-normalization mapping. */ const ScanKeyword* ScanKeywordLookup(const char* text, const ScanKeyword* keywords, int num_keywords) -{ +{ /* + * @param[IN] text：The identifier to be matched. + * @param[IN] keywords：A pointer to the keyword table. + * @param[IN] num_keywords: The number of keywords table items (keywords). + * @param[OUT]：A pointer to a keywords table item. Null is returned if the match fails. + */ + int len, i; - char word[NAMEDATALEN] = {0}; - const ScanKeyword* low = NULL; - const ScanKeyword* high = NULL; + char word[NAMEDATALEN] = {0}; + const ScanKeyword* low = NULL; // Pointer used in the binary search. + const ScanKeyword* high = NULL; // Pointer used in the binary search. - if (text == NULL) { + // The input is NULL, no match is made, and NULL is returned. + if (text == NULL) { return NULL; } @@ -64,7 +71,7 @@ const ScanKeyword* ScanKeywordLookup(const char* text, const ScanKeyword* keywor } word[i] = ch; } - word[len] = '\0'; + word[len] = '\0'; // The converted text should ends with '\0'. /* * Now do a binary search using plain strcmp() comparison. @@ -85,6 +92,6 @@ const ScanKeyword* ScanKeywordLookup(const char* text, const ScanKeyword* keywor high = middle - 1; } } - + // The binary search fails and returns a null value. return NULL; -} +} \ No newline at end of file -- 2.34.1 From 0bda37330c6c6098a1458c2e011153ac17da7c65 Mon Sep 17 00:00:00 2001 From: XL_up <3066352084@qq.com> Date: Mon, 28 Aug 2023 22:59:30 +0800 Subject: [PATCH 4/9] Update kwlookup.cpp --- src/common/backend/parser/kwlookup.cpp | 629 ++++++++++++++++++++++--- 1 file changed, 564 insertions(+), 65 deletions(-) diff --git a/src/common/backend/parser/kwlookup.cpp b/src/common/backend/parser/kwlookup.cpp index 899c96a1..d7541e83 100644 --- a/src/common/backend/parser/kwlookup.cpp +++ b/src/common/backend/parser/kwlookup.cpp @@ -1,97 +1,596 @@ /* ------------------------------------------------------------------------- * - * kwlookup.cpp - * lexical token lookup for key words in openGauss + * parser.cpp + * Main entry point/driver for openGauss grammar * - * NB - this file is also used by ECPG and several frontend programs in - * src/bin/ including pg_dump and psql + * Note that the grammar is not allowed to perform any table access + * (since we need to be able to do basic parsing even while inside an + * aborted transaction). Therefore, the data structures returned by + * the grammar are "raw" parsetrees that still need to be analyzed by + * analyze.c and related files. * * Portions Copyright (c) 1996-2012, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * * IDENTIFICATION - * src/common/backend/parser/kwlookup.cpp + * src/common/backend/parser/parser.cpp * * ------------------------------------------------------------------------- */ -/* use c.h so this can be built as either frontend or backend */ -#include "c.h" +#include "postgres.h" +#include "knl/knl_variable.h" +#include "nodes/parsenodes.h" -#include +#include "parser/gramparse.h" +#include "parser/parser.h" -#include "parser/keywords.h" +extern void resetOperatorPlusFlag(); + +static void resetIsTimeCapsuleFlag() +{ + u_sess->parser_cxt.isTimeCapsule = false; +} + +static void resetCreateFuncFlag() +{ + u_sess->parser_cxt.isCreateFuncOrProc = false; +} /* - * ScanKeywordLookup - see if a given word is a keyword + * raw_parser + * Given a query in string form, do lexical and grammatical analysis. * - * Returns a pointer to the ScanKeyword table entry, or NULL if no match. - * - * The match is done case-insensitively. Note that we deliberately use a - * dumbed-down case conversion that will only translate 'A'-'Z' into 'a'-'z', - * even if we are in a locale where tolower() would produce more or different - * translations. This is to conform to the SQL99 spec, which says that - * keywords are to be matched in this way even though non-keyword identifiers - * receive a different case-normalization mapping. + * Returns a list of raw (un-analyzed) parse trees. */ -const ScanKeyword* ScanKeywordLookup(const char* text, const ScanKeyword* keywords, int num_keywords) -{ /* - * @param[IN] text：The identifier to be matched. - * @param[IN] keywords：A pointer to the keyword table. - * @param[IN] num_keywords: The number of keywords table items (keywords). - * @param[OUT]：A pointer to a keywords table item. Null is returned if the match fails. - */ +List* raw_parser(const char* str, List** query_string_locationlist) +{ + core_yyscan_t yyscanner; + base_yy_extra_type yyextra; // An intermediate variable that holds the returned syntax tree. + int yyresult; // The calling result of base_yyparse(). + + /* reset u_sess->parser_cxt.stmt_contains_operator_plus */ + resetOperatorPlusFlag(); + + /* reset u_sess->parser_cxt.isTimeCapsule */ + resetIsTimeCapsuleFlag(); + + /* reset u_sess->parser_cxt.isCreateFuncOrProc */ + resetCreateFuncFlag(); - int len, i; - char word[NAMEDATALEN] = {0}; - const ScanKeyword* low = NULL; // Pointer used in the binary search. - const ScanKeyword* high = NULL; // Pointer used in the binary search. + /* initialize the flex scanner */ + yyscanner = scanner_init(str, &yyextra.core_yy_extra, ScanKeywords, NumScanKeywords); - // The input is NULL, no match is made, and NULL is returned. - if (text == NULL) { - return NULL; + /* base_yylex() only needs this much initialization */ + yyextra.lookahead_num = 0; + + /* initialize the bison parser */ + parser_init(&yyextra); + + /* Parse! */ + yyresult = base_yyparse(yyscanner); + + /* Clean up (release memory) */ + scanner_finish(yyscanner); + + if (yyresult) { /* error */ + return NIL; } - len = strlen(text); - /* We assume all keywords are shorter than NAMEDATALEN. */ - if (len >= NAMEDATALEN) { - return NULL; - } + /* Get the locationlist of multi-query through lex. */ + if (query_string_locationlist != NULL) { + *query_string_locationlist = yyextra.core_yy_extra.query_string_locationlist; - /* - * Apply an ASCII-only downcasing. We must not use tolower() since it may - * produce the wrong translation in some locales (eg, Turkish). - */ - for (i = 0; i < len; i++) { - char ch = text[i]; - - if (ch >= 'A' && ch <= 'Z') { - ch += 'a' - 'A'; + /* Deal with the query sent from client without semicolon at the end. */ + if (PointerIsValid(*query_string_locationlist) && + (size_t)lfirst_int(list_tail(*query_string_locationlist)) < (strlen(str) - 1)) { + *query_string_locationlist = lappend_int(*query_string_locationlist, strlen(str)); } - word[i] = ch; } - word[len] = '\0'; // The converted text should ends with '\0'. + + // Returns the generated syntax tree. + return yyextra.parsetree; +} + +#define GET_NEXT_TOKEN() \ + do { \ + cur_yylval = lvalp->core_yystype; \ + cur_yylloc = *llocp; \ + if (yyextra->lookahead_num != 0) { \ + next_token = yyextra->lookahead_token[yyextra->lookahead_num - 1]; \ + lvalp->core_yystype = yyextra->lookahead_yylval[yyextra->lookahead_num - 1]; \ + *llocp = yyextra->lookahead_yylloc[yyextra->lookahead_num - 1]; \ + yyextra->lookahead_num--; \ + Assert(yyextra->lookahead_num == 0); \ + } else { \ + next_token = core_yylex(&(lvalp->core_yystype), llocp, yyscanner); \ + } \ + } while (0) + +#define SET_LOOKAHEAD_TOKEN() \ + do { \ + yyextra->lookahead_token[0] = next_token; \ + yyextra->lookahead_yylval[0] = lvalp->core_yystype; \ + yyextra->lookahead_yylloc[0] = *llocp; \ + yyextra->lookahead_num = 1; \ + } while (0) + +/* + * Intermediate filter between parser and core lexer (core_yylex in scan.l). + * + * The filter is needed because in some cases the standard SQL grammar + * requires more than one token lookahead. We reduce these cases to one-token + * lookahead by combining tokens here, in order to keep the grammar LALR(1). + * + * Using a filter is simpler than trying to recognize multiword tokens + * directly in scan.l, because we'd have to allow for comments between the + * words. Furthermore it's not clear how to do it without re-introducing + * scanner backtrack, which would cost more performance than this filter + * layer does. + * + * The filter also provides a convenient place to translate between + * the core_YYSTYPE and YYSTYPE representations (which are really the + * same thing anyway, but notationally they're different). + */ +int base_yylex(YYSTYPE* lvalp, YYLTYPE* llocp, core_yyscan_t yyscanner) +{ + base_yy_extra_type* yyextra = pg_yyget_extra(yyscanner); + int cur_token; + int next_token; + core_YYSTYPE cur_yylval; + YYLTYPE cur_yylloc; + + /* Get next token --- we might already have it */ + if (yyextra->lookahead_num != 0) { + cur_token = yyextra->lookahead_token[yyextra->lookahead_num - 1]; + lvalp->core_yystype = yyextra->lookahead_yylval[yyextra->lookahead_num - 1]; + *llocp = yyextra->lookahead_yylloc[yyextra->lookahead_num - 1]; + yyextra->lookahead_num--; + } else { + cur_token = core_yylex(&(lvalp->core_yystype), llocp, yyscanner); + } + + /* Do we need to look ahead for a possible multiword token? */ + switch (cur_token) { + case NULLS_P: + /* + * NULLS FIRST and NULLS LAST must be reduced to one token + */ + GET_NEXT_TOKEN(); + switch (next_token) { + case FIRST_P: + cur_token = NULLS_FIRST; + break; + case LAST_P: + cur_token = NULLS_LAST; + break; + default: + /* save the lookahead token for next time */ + SET_LOOKAHEAD_TOKEN(); + /* and back up the output info to cur_token */ + lvalp->core_yystype = cur_yylval; + *llocp = cur_yylloc; + break; + } + break; + + case NOT: + /* + * @hdfs + * In order to solve the conflict in gram.y, NOT and ENFORCED must be reduced to one token. + */ + GET_NEXT_TOKEN(); + + switch (next_token) { + case ENFORCED: + cur_token = NOT_ENFORCED; + break; + default: + /* save the lookahead token for next time */ + SET_LOOKAHEAD_TOKEN(); + + /* and back up the output info to cur_token */ + lvalp->core_yystype = cur_yylval; + *llocp = cur_yylloc; + break; + } + break; + + case WITH: + /* + * WITH TIME must be reduced to one token + */ + GET_NEXT_TOKEN(); + + switch (next_token) { + case TIME: + cur_token = WITH_TIME; + break; + default: + /* save the lookahead token for next time */ + SET_LOOKAHEAD_TOKEN(); + + /* and back up the output info to cur_token */ + lvalp->core_yystype = cur_yylval; + *llocp = cur_yylloc; + break; + } + break; + + case INCLUDING: + + /* + * INCLUDING ALL must be reduced to one token + */ + GET_NEXT_TOKEN(); + + switch (next_token) { + case ALL: + cur_token = INCLUDING_ALL; + break; + default: + /* save the lookahead token for next time */ + SET_LOOKAHEAD_TOKEN(); + + /* and back up the output info to cur_token */ + lvalp->core_yystype = cur_yylval; + *llocp = cur_yylloc; + break; + } + break; + + case RENAME: + + /* + * RENAME PARTITION must be reduced to one token + */ + GET_NEXT_TOKEN(); + + switch (next_token) { + case PARTITION: + cur_token = RENAME_PARTITION; + break; + default: + /* save the lookahead token for next time */ + SET_LOOKAHEAD_TOKEN(); + /* and back up the output info to cur_token */ + lvalp->core_yystype = cur_yylval; + *llocp = cur_yylloc; + break; + } + break; + + case PARTITION: + + /* + * RENAME PARTITION must be reduced to one token + */ + GET_NEXT_TOKEN(); + + switch (next_token) { + case FOR: + cur_token = PARTITION_FOR; + break; + default: + /* save the lookahead token for next time */ + SET_LOOKAHEAD_TOKEN(); + /* and back up the output info to cur_token */ + lvalp->core_yystype = cur_yylval; + *llocp = cur_yylloc; + break; + } + break; + case SUBPARTITION: + + GET_NEXT_TOKEN(); + + switch (next_token) { + case FOR: + cur_token = SUBPARTITION_FOR; + break; + default: + /* save the lookahead token for next time */ + SET_LOOKAHEAD_TOKEN(); + /* and back up the output info to cur_token */ + lvalp->core_yystype = cur_yylval; + *llocp = cur_yylloc; + break; + } + break; + case ADD_P: + /* + * ADD PARTITION must be reduced to one token + */ + GET_NEXT_TOKEN(); + + switch (next_token) { + case PARTITION: + cur_token = ADD_PARTITION; + break; + case SUBPARTITION: + cur_token = ADD_SUBPARTITION; + break; + default: + /* save the lookahead token for next time */ + SET_LOOKAHEAD_TOKEN(); + /* and back up the output info to cur_token */ + lvalp->core_yystype = cur_yylval; + *llocp = cur_yylloc; + break; + } + break; + + case DROP: + + /* + * DROP PARTITION must be reduced to one token + */ + GET_NEXT_TOKEN(); + + switch (next_token) { + case PARTITION: + cur_token = DROP_PARTITION; + break; + case SUBPARTITION: + cur_token = DROP_SUBPARTITION; + break; + default: + /* save the lookahead token for next time */ + SET_LOOKAHEAD_TOKEN(); + /* and back up the output info to cur_token */ + lvalp->core_yystype = cur_yylval; + *llocp = cur_yylloc; + break; + } + break; + case REBUILD: + + /* + * REBUILD PARTITION must be reduced to one token + */ + GET_NEXT_TOKEN(); + + switch (next_token) { + case PARTITION: + cur_token = REBUILD_PARTITION; + break; + default: + /* save the lookahead token for next time */ + SET_LOOKAHEAD_TOKEN(); + /* and back up the output info to cur_token */ + lvalp->core_yystype = cur_yylval; + *llocp = cur_yylloc; + break; + } + break; + case MODIFY_P: + /* + * MODIFY PARTITION must be reduced to one token + */ + GET_NEXT_TOKEN(); + + switch (next_token) { + case PARTITION: + cur_token = MODIFY_PARTITION; + break; + default: + /* save the lookahead token for next time */ + SET_LOOKAHEAD_TOKEN(); + /* and back up the output info to cur_token */ + lvalp->core_yystype = cur_yylval; + *llocp = cur_yylloc; + break; + } + break; + case DECLARE: + /* + * DECLARE foo CUROSR must be looked ahead, and if determined as a DECLARE_CURSOR, we should set the yylaval + * and yylloc back, letting the parser read the cursor name correctly. + */ + cur_yylval = lvalp->core_yystype; + cur_yylloc = *llocp; + next_token = core_yylex(&(lvalp->core_yystype), llocp, yyscanner); + /* get first token after DECLARE. We don't care what it is */ + yyextra->lookahead_token[1] = next_token; + yyextra->lookahead_yylval[1] = lvalp->core_yystype; + yyextra->lookahead_yylloc[1] = *llocp; + + /* get the second token after DECLARE. If it is cursor grammer, we are sure that this is a cursr stmt */ + next_token = core_yylex(&(lvalp->core_yystype), llocp, yyscanner); + yyextra->lookahead_token[0] = next_token; + yyextra->lookahead_yylval[0] = lvalp->core_yystype; + yyextra->lookahead_yylloc[0] = *llocp; + yyextra->lookahead_num = 2; + + switch (next_token) { + case CURSOR: + case BINARY: + case INSENSITIVE: + case NO: + case SCROLL: + cur_token = DECLARE_CURSOR; + /* and back up the output info to cur_token because we should read cursor name correctly. */ + lvalp->core_yystype = cur_yylval; + *llocp = cur_yylloc; + break; + default: + /* and back up the output info to cur_token */ + lvalp->core_yystype = cur_yylval; + *llocp = cur_yylloc; + break; + } + break; + case VALID: + + /* + * VALID BEGIN must be reduced to one token, to avoid conflict with BEGIN TRANSACTIOn and BEGIN anonymous + * block. + */ + GET_NEXT_TOKEN(); + + switch (next_token) { + case BEGIN_P: + case BEGIN_NON_ANOYBLOCK: + cur_token = VALID_BEGIN; + break; + default: + /* save the lookahead token for next time */ + SET_LOOKAHEAD_TOKEN(); + /* and back up the output info to cur_token */ + lvalp->core_yystype = cur_yylval; + *llocp = cur_yylloc; + break; + } + break; + case START: + /* + * START WITH must be reduced to one token, to allow START as table / column alias. + */ + GET_NEXT_TOKEN(); + + switch (next_token) { + case WITH: + cur_token = START_WITH; + break; + default: + /* save the lookahead token for next time */ + SET_LOOKAHEAD_TOKEN(); + /* and back up the output info to cur_token */ + lvalp->core_yystype = cur_yylval; + *llocp = cur_yylloc; + break; + } + break; + case CONNECT: + /* + * CONNECT BY must be reduced to one token, to allow CONNECT as table / column alias. + */ + GET_NEXT_TOKEN(); + + switch (next_token) { + case BY: + cur_token = CONNECT_BY; + break; + default: + /* save the lookahead token for next time */ + SET_LOOKAHEAD_TOKEN(); + /* and back up the output info to cur_token */ + lvalp->core_yystype = cur_yylval; + *llocp = cur_yylloc; + break; + } + break; + default: + break; + } + + return cur_token; +} + +/* + * @Description: Check whether its a empty query with only comments and semicolon. + * @Param[IN] query_string: the query need check. + * @return:the bool value of the check result. + */ +static bool is_empty_query(char* query_string) +{ + char begin_comment[3] = "/*"; + char end_comment[3] = "*/"; + char empty_query[2] = ";"; + char* end_comment_postion = NULL; + + /* Trim all the spaces at the begin of the string. */ + while (isspace((unsigned char)*query_string)) { + query_string++; + } + + /* Trim all the comments of the query_string from the front. */ + while (strncmp(query_string, begin_comment, 2) == 0) { + /* + * As query_string have been through parser, whenever it contain the begin_comment + * it will comtain the end_comment and end_comment_postion can't be null here. + */ + end_comment_postion = strstr(query_string, end_comment); + query_string = end_comment_postion + 2; + while (isspace((unsigned char)*query_string)) { + // Trim the spaces after comments. + query_string++; + } + } + + /* Check whether query_string is a empty query. */ + if (strcmp(query_string, empty_query) == 0) { + return true; + } else { + return false; + } +} + +/* + * @Description: split the query_string to distinct single querys. + * @Param [IN] query_string_single: store the splited single querys. + * @Param [IN] query_string: initial query string which contain multi statements. + * @Param [IN] query_string_locationList: record single query terminator-semicolon locations which get from lexer. + * @Param [IN] stmt_num: show this is the n-ths single query of the multi query. + * @return [IN/OUT] query_string_single: store the point arrary of single query. + * @NOTICE:The caller is responsible for freeing the storage palloced here. + */ +char** get_next_snippet( + char** query_string_single, const char* query_string, List* query_string_locationlist, int* stmt_num) +{ + int query_string_location_start = 0; // The starting position of the query statement. + int query_string_location_end = -1; // The ending position of the query statement. + char* query_string_single_p = NULL; // An intermediate variable used to copy the string. + int single_query_string_len = 0; // The length of the query statement. + + // Count the number of query statements. + int stmt_count = list_length(query_string_locationlist); + + /* Malloc memory for single query here just for the first time. */ + if (query_string_single == NULL) { + query_string_single = (char**)palloc0(sizeof(char*) * stmt_count); + } /* - * Now do a binary search using plain strcmp() comparison. + * Get the snippet of multi_query until we get a non-empty query as the empty query string + * needn't be dealed with. */ - low = keywords; - high = keywords + (num_keywords - 1); - while (low <= high) { - const ScanKeyword* middle = NULL; - int difference; - - middle = low + (high - low) / 2; - difference = strcmp(middle->name, word); - if (difference == 0) { - return middle; - } else if (difference < 0) { - low = middle + 1; + for (; *stmt_num < stmt_count;) { + /* + * Notice : The locationlist only store the end postion of each single query but not any + * start postion. + */ + // Calculate the starting position of the specified query statement. + if (*stmt_num == 0) { + query_string_location_start = 0; } else { - high = middle - 1; + query_string_location_start = list_nth_int(query_string_locationlist, *stmt_num - 1) + 1; + } + // Calculate the ending position of the specified query statement. + query_string_location_end = list_nth_int(query_string_locationlist, (*stmt_num)++); + + /* Malloc memory for each single query string. */ + single_query_string_len = query_string_location_end - query_string_location_start + 1; + query_string_single[*stmt_num - 1] = (char*)palloc0(sizeof(char) * (single_query_string_len + 1)); + + /* Copy the query_string between location_start and location_end to query_string_single. */ + query_string_single_p = query_string_single[*stmt_num - 1]; + while (query_string_location_start <= query_string_location_end) { + *query_string_single_p = *(query_string + query_string_location_start); + query_string_location_start++; + query_string_single_p++; + } + + /* + * If query_string_single is empty query which only contain comments or null strings, + * we will skip it. + */ + if (is_empty_query(query_string_single[*stmt_num - 1])) { + continue; + } else { // The obtained query statement is not a null query, exit the loop, and return this statement. + break; } } - // The binary search fails and returns a null value. - return NULL; + + return query_string_single; } \ No newline at end of file -- 2.34.1 From 08f23c214845683cd3490097d4ab71f0ffc22588 Mon Sep 17 00:00:00 2001 From: XL_up <3066352084@qq.com> Date: Mon, 28 Aug 2023 23:00:07 +0800 Subject: [PATCH 5/9] Update kwlookup.cpp --- src/common/backend/parser/kwlookup.cpp | 633 +++---------------------- 1 file changed, 67 insertions(+), 566 deletions(-) diff --git a/src/common/backend/parser/kwlookup.cpp b/src/common/backend/parser/kwlookup.cpp index d7541e83..899c96a1 100644 --- a/src/common/backend/parser/kwlookup.cpp +++ b/src/common/backend/parser/kwlookup.cpp @@ -1,596 +1,97 @@ /* ------------------------------------------------------------------------- * - * parser.cpp - * Main entry point/driver for openGauss grammar + * kwlookup.cpp + * lexical token lookup for key words in openGauss * - * Note that the grammar is not allowed to perform any table access - * (since we need to be able to do basic parsing even while inside an - * aborted transaction). Therefore, the data structures returned by - * the grammar are "raw" parsetrees that still need to be analyzed by - * analyze.c and related files. + * NB - this file is also used by ECPG and several frontend programs in + * src/bin/ including pg_dump and psql * * Portions Copyright (c) 1996-2012, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * + * * IDENTIFICATION - * src/common/backend/parser/parser.cpp + * src/common/backend/parser/kwlookup.cpp * * ------------------------------------------------------------------------- */ -#include "postgres.h" -#include "knl/knl_variable.h" -#include "nodes/parsenodes.h" +/* use c.h so this can be built as either frontend or backend */ +#include "c.h" -#include "parser/gramparse.h" -#include "parser/parser.h" +#include -extern void resetOperatorPlusFlag(); - -static void resetIsTimeCapsuleFlag() -{ - u_sess->parser_cxt.isTimeCapsule = false; -} - -static void resetCreateFuncFlag() -{ - u_sess->parser_cxt.isCreateFuncOrProc = false; -} +#include "parser/keywords.h" /* - * raw_parser - * Given a query in string form, do lexical and grammatical analysis. + * ScanKeywordLookup - see if a given word is a keyword * - * Returns a list of raw (un-analyzed) parse trees. + * Returns a pointer to the ScanKeyword table entry, or NULL if no match. + * + * The match is done case-insensitively. Note that we deliberately use a + * dumbed-down case conversion that will only translate 'A'-'Z' into 'a'-'z', + * even if we are in a locale where tolower() would produce more or different + * translations. This is to conform to the SQL99 spec, which says that + * keywords are to be matched in this way even though non-keyword identifiers + * receive a different case-normalization mapping. */ -List* raw_parser(const char* str, List** query_string_locationlist) -{ - core_yyscan_t yyscanner; - base_yy_extra_type yyextra; // An intermediate variable that holds the returned syntax tree. - int yyresult; // The calling result of base_yyparse(). - - /* reset u_sess->parser_cxt.stmt_contains_operator_plus */ - resetOperatorPlusFlag(); - - /* reset u_sess->parser_cxt.isTimeCapsule */ - resetIsTimeCapsuleFlag(); - - /* reset u_sess->parser_cxt.isCreateFuncOrProc */ - resetCreateFuncFlag(); +const ScanKeyword* ScanKeywordLookup(const char* text, const ScanKeyword* keywords, int num_keywords) +{ /* + * @param[IN] text：The identifier to be matched. + * @param[IN] keywords：A pointer to the keyword table. + * @param[IN] num_keywords: The number of keywords table items (keywords). + * @param[OUT]：A pointer to a keywords table item. Null is returned if the match fails. + */ - /* initialize the flex scanner */ - yyscanner = scanner_init(str, &yyextra.core_yy_extra, ScanKeywords, NumScanKeywords); + int len, i; + char word[NAMEDATALEN] = {0}; + const ScanKeyword* low = NULL; // Pointer used in the binary search. + const ScanKeyword* high = NULL; // Pointer used in the binary search. - /* base_yylex() only needs this much initialization */ - yyextra.lookahead_num = 0; - - /* initialize the bison parser */ - parser_init(&yyextra); - - /* Parse! */ - yyresult = base_yyparse(yyscanner); - - /* Clean up (release memory) */ - scanner_finish(yyscanner); - - if (yyresult) { /* error */ - return NIL; + // The input is NULL, no match is made, and NULL is returned. + if (text == NULL) { + return NULL; } - /* Get the locationlist of multi-query through lex. */ - if (query_string_locationlist != NULL) { - *query_string_locationlist = yyextra.core_yy_extra.query_string_locationlist; - - /* Deal with the query sent from client without semicolon at the end. */ - if (PointerIsValid(*query_string_locationlist) && - (size_t)lfirst_int(list_tail(*query_string_locationlist)) < (strlen(str) - 1)) { - *query_string_locationlist = lappend_int(*query_string_locationlist, strlen(str)); - } - } - - // Returns the generated syntax tree. - return yyextra.parsetree; -} - -#define GET_NEXT_TOKEN() \ - do { \ - cur_yylval = lvalp->core_yystype; \ - cur_yylloc = *llocp; \ - if (yyextra->lookahead_num != 0) { \ - next_token = yyextra->lookahead_token[yyextra->lookahead_num - 1]; \ - lvalp->core_yystype = yyextra->lookahead_yylval[yyextra->lookahead_num - 1]; \ - *llocp = yyextra->lookahead_yylloc[yyextra->lookahead_num - 1]; \ - yyextra->lookahead_num--; \ - Assert(yyextra->lookahead_num == 0); \ - } else { \ - next_token = core_yylex(&(lvalp->core_yystype), llocp, yyscanner); \ - } \ - } while (0) - -#define SET_LOOKAHEAD_TOKEN() \ - do { \ - yyextra->lookahead_token[0] = next_token; \ - yyextra->lookahead_yylval[0] = lvalp->core_yystype; \ - yyextra->lookahead_yylloc[0] = *llocp; \ - yyextra->lookahead_num = 1; \ - } while (0) - -/* - * Intermediate filter between parser and core lexer (core_yylex in scan.l). - * - * The filter is needed because in some cases the standard SQL grammar - * requires more than one token lookahead. We reduce these cases to one-token - * lookahead by combining tokens here, in order to keep the grammar LALR(1). - * - * Using a filter is simpler than trying to recognize multiword tokens - * directly in scan.l, because we'd have to allow for comments between the - * words. Furthermore it's not clear how to do it without re-introducing - * scanner backtrack, which would cost more performance than this filter - * layer does. - * - * The filter also provides a convenient place to translate between - * the core_YYSTYPE and YYSTYPE representations (which are really the - * same thing anyway, but notationally they're different). - */ -int base_yylex(YYSTYPE* lvalp, YYLTYPE* llocp, core_yyscan_t yyscanner) -{ - base_yy_extra_type* yyextra = pg_yyget_extra(yyscanner); - int cur_token; - int next_token; - core_YYSTYPE cur_yylval; - YYLTYPE cur_yylloc; - - /* Get next token --- we might already have it */ - if (yyextra->lookahead_num != 0) { - cur_token = yyextra->lookahead_token[yyextra->lookahead_num - 1]; - lvalp->core_yystype = yyextra->lookahead_yylval[yyextra->lookahead_num - 1]; - *llocp = yyextra->lookahead_yylloc[yyextra->lookahead_num - 1]; - yyextra->lookahead_num--; - } else { - cur_token = core_yylex(&(lvalp->core_yystype), llocp, yyscanner); - } - - /* Do we need to look ahead for a possible multiword token? */ - switch (cur_token) { - case NULLS_P: - /* - * NULLS FIRST and NULLS LAST must be reduced to one token - */ - GET_NEXT_TOKEN(); - switch (next_token) { - case FIRST_P: - cur_token = NULLS_FIRST; - break; - case LAST_P: - cur_token = NULLS_LAST; - break; - default: - /* save the lookahead token for next time */ - SET_LOOKAHEAD_TOKEN(); - /* and back up the output info to cur_token */ - lvalp->core_yystype = cur_yylval; - *llocp = cur_yylloc; - break; - } - break; - - case NOT: - /* - * @hdfs - * In order to solve the conflict in gram.y, NOT and ENFORCED must be reduced to one token. - */ - GET_NEXT_TOKEN(); - - switch (next_token) { - case ENFORCED: - cur_token = NOT_ENFORCED; - break; - default: - /* save the lookahead token for next time */ - SET_LOOKAHEAD_TOKEN(); - - /* and back up the output info to cur_token */ - lvalp->core_yystype = cur_yylval; - *llocp = cur_yylloc; - break; - } - break; - - case WITH: - /* - * WITH TIME must be reduced to one token - */ - GET_NEXT_TOKEN(); - - switch (next_token) { - case TIME: - cur_token = WITH_TIME; - break; - default: - /* save the lookahead token for next time */ - SET_LOOKAHEAD_TOKEN(); - - /* and back up the output info to cur_token */ - lvalp->core_yystype = cur_yylval; - *llocp = cur_yylloc; - break; - } - break; - - case INCLUDING: - - /* - * INCLUDING ALL must be reduced to one token - */ - GET_NEXT_TOKEN(); - - switch (next_token) { - case ALL: - cur_token = INCLUDING_ALL; - break; - default: - /* save the lookahead token for next time */ - SET_LOOKAHEAD_TOKEN(); - - /* and back up the output info to cur_token */ - lvalp->core_yystype = cur_yylval; - *llocp = cur_yylloc; - break; - } - break; - - case RENAME: - - /* - * RENAME PARTITION must be reduced to one token - */ - GET_NEXT_TOKEN(); - - switch (next_token) { - case PARTITION: - cur_token = RENAME_PARTITION; - break; - default: - /* save the lookahead token for next time */ - SET_LOOKAHEAD_TOKEN(); - /* and back up the output info to cur_token */ - lvalp->core_yystype = cur_yylval; - *llocp = cur_yylloc; - break; - } - break; - - case PARTITION: - - /* - * RENAME PARTITION must be reduced to one token - */ - GET_NEXT_TOKEN(); - - switch (next_token) { - case FOR: - cur_token = PARTITION_FOR; - break; - default: - /* save the lookahead token for next time */ - SET_LOOKAHEAD_TOKEN(); - /* and back up the output info to cur_token */ - lvalp->core_yystype = cur_yylval; - *llocp = cur_yylloc; - break; - } - break; - case SUBPARTITION: - - GET_NEXT_TOKEN(); - - switch (next_token) { - case FOR: - cur_token = SUBPARTITION_FOR; - break; - default: - /* save the lookahead token for next time */ - SET_LOOKAHEAD_TOKEN(); - /* and back up the output info to cur_token */ - lvalp->core_yystype = cur_yylval; - *llocp = cur_yylloc; - break; - } - break; - case ADD_P: - /* - * ADD PARTITION must be reduced to one token - */ - GET_NEXT_TOKEN(); - - switch (next_token) { - case PARTITION: - cur_token = ADD_PARTITION; - break; - case SUBPARTITION: - cur_token = ADD_SUBPARTITION; - break; - default: - /* save the lookahead token for next time */ - SET_LOOKAHEAD_TOKEN(); - /* and back up the output info to cur_token */ - lvalp->core_yystype = cur_yylval; - *llocp = cur_yylloc; - break; - } - break; - - case DROP: - - /* - * DROP PARTITION must be reduced to one token - */ - GET_NEXT_TOKEN(); - - switch (next_token) { - case PARTITION: - cur_token = DROP_PARTITION; - break; - case SUBPARTITION: - cur_token = DROP_SUBPARTITION; - break; - default: - /* save the lookahead token for next time */ - SET_LOOKAHEAD_TOKEN(); - /* and back up the output info to cur_token */ - lvalp->core_yystype = cur_yylval; - *llocp = cur_yylloc; - break; - } - break; - case REBUILD: - - /* - * REBUILD PARTITION must be reduced to one token - */ - GET_NEXT_TOKEN(); - - switch (next_token) { - case PARTITION: - cur_token = REBUILD_PARTITION; - break; - default: - /* save the lookahead token for next time */ - SET_LOOKAHEAD_TOKEN(); - /* and back up the output info to cur_token */ - lvalp->core_yystype = cur_yylval; - *llocp = cur_yylloc; - break; - } - break; - case MODIFY_P: - /* - * MODIFY PARTITION must be reduced to one token - */ - GET_NEXT_TOKEN(); - - switch (next_token) { - case PARTITION: - cur_token = MODIFY_PARTITION; - break; - default: - /* save the lookahead token for next time */ - SET_LOOKAHEAD_TOKEN(); - /* and back up the output info to cur_token */ - lvalp->core_yystype = cur_yylval; - *llocp = cur_yylloc; - break; - } - break; - case DECLARE: - /* - * DECLARE foo CUROSR must be looked ahead, and if determined as a DECLARE_CURSOR, we should set the yylaval - * and yylloc back, letting the parser read the cursor name correctly. - */ - cur_yylval = lvalp->core_yystype; - cur_yylloc = *llocp; - next_token = core_yylex(&(lvalp->core_yystype), llocp, yyscanner); - /* get first token after DECLARE. We don't care what it is */ - yyextra->lookahead_token[1] = next_token; - yyextra->lookahead_yylval[1] = lvalp->core_yystype; - yyextra->lookahead_yylloc[1] = *llocp; - - /* get the second token after DECLARE. If it is cursor grammer, we are sure that this is a cursr stmt */ - next_token = core_yylex(&(lvalp->core_yystype), llocp, yyscanner); - yyextra->lookahead_token[0] = next_token; - yyextra->lookahead_yylval[0] = lvalp->core_yystype; - yyextra->lookahead_yylloc[0] = *llocp; - yyextra->lookahead_num = 2; - - switch (next_token) { - case CURSOR: - case BINARY: - case INSENSITIVE: - case NO: - case SCROLL: - cur_token = DECLARE_CURSOR; - /* and back up the output info to cur_token because we should read cursor name correctly. */ - lvalp->core_yystype = cur_yylval; - *llocp = cur_yylloc; - break; - default: - /* and back up the output info to cur_token */ - lvalp->core_yystype = cur_yylval; - *llocp = cur_yylloc; - break; - } - break; - case VALID: - - /* - * VALID BEGIN must be reduced to one token, to avoid conflict with BEGIN TRANSACTIOn and BEGIN anonymous - * block. - */ - GET_NEXT_TOKEN(); - - switch (next_token) { - case BEGIN_P: - case BEGIN_NON_ANOYBLOCK: - cur_token = VALID_BEGIN; - break; - default: - /* save the lookahead token for next time */ - SET_LOOKAHEAD_TOKEN(); - /* and back up the output info to cur_token */ - lvalp->core_yystype = cur_yylval; - *llocp = cur_yylloc; - break; - } - break; - case START: - /* - * START WITH must be reduced to one token, to allow START as table / column alias. - */ - GET_NEXT_TOKEN(); - - switch (next_token) { - case WITH: - cur_token = START_WITH; - break; - default: - /* save the lookahead token for next time */ - SET_LOOKAHEAD_TOKEN(); - /* and back up the output info to cur_token */ - lvalp->core_yystype = cur_yylval; - *llocp = cur_yylloc; - break; - } - break; - case CONNECT: - /* - * CONNECT BY must be reduced to one token, to allow CONNECT as table / column alias. - */ - GET_NEXT_TOKEN(); - - switch (next_token) { - case BY: - cur_token = CONNECT_BY; - break; - default: - /* save the lookahead token for next time */ - SET_LOOKAHEAD_TOKEN(); - /* and back up the output info to cur_token */ - lvalp->core_yystype = cur_yylval; - *llocp = cur_yylloc; - break; - } - break; - default: - break; - } - - return cur_token; -} - -/* - * @Description: Check whether its a empty query with only comments and semicolon. - * @Param[IN] query_string: the query need check. - * @return:the bool value of the check result. - */ -static bool is_empty_query(char* query_string) -{ - char begin_comment[3] = "/*"; - char end_comment[3] = "*/"; - char empty_query[2] = ";"; - char* end_comment_postion = NULL; - - /* Trim all the spaces at the begin of the string. */ - while (isspace((unsigned char)*query_string)) { - query_string++; - } - - /* Trim all the comments of the query_string from the front. */ - while (strncmp(query_string, begin_comment, 2) == 0) { - /* - * As query_string have been through parser, whenever it contain the begin_comment - * it will comtain the end_comment and end_comment_postion can't be null here. - */ - end_comment_postion = strstr(query_string, end_comment); - query_string = end_comment_postion + 2; - while (isspace((unsigned char)*query_string)) { - // Trim the spaces after comments. - query_string++; - } - } - - /* Check whether query_string is a empty query. */ - if (strcmp(query_string, empty_query) == 0) { - return true; - } else { - return false; - } -} - -/* - * @Description: split the query_string to distinct single querys. - * @Param [IN] query_string_single: store the splited single querys. - * @Param [IN] query_string: initial query string which contain multi statements. - * @Param [IN] query_string_locationList: record single query terminator-semicolon locations which get from lexer. - * @Param [IN] stmt_num: show this is the n-ths single query of the multi query. - * @return [IN/OUT] query_string_single: store the point arrary of single query. - * @NOTICE:The caller is responsible for freeing the storage palloced here. - */ -char** get_next_snippet( - char** query_string_single, const char* query_string, List* query_string_locationlist, int* stmt_num) -{ - int query_string_location_start = 0; // The starting position of the query statement. - int query_string_location_end = -1; // The ending position of the query statement. - char* query_string_single_p = NULL; // An intermediate variable used to copy the string. - int single_query_string_len = 0; // The length of the query statement. - - // Count the number of query statements. - int stmt_count = list_length(query_string_locationlist); - - /* Malloc memory for single query here just for the first time. */ - if (query_string_single == NULL) { - query_string_single = (char**)palloc0(sizeof(char*) * stmt_count); + len = strlen(text); + /* We assume all keywords are shorter than NAMEDATALEN. */ + if (len >= NAMEDATALEN) { + return NULL; } /* - * Get the snippet of multi_query until we get a non-empty query as the empty query string - * needn't be dealed with. + * Apply an ASCII-only downcasing. We must not use tolower() since it may + * produce the wrong translation in some locales (eg, Turkish). */ - for (; *stmt_num < stmt_count;) { - /* - * Notice : The locationlist only store the end postion of each single query but not any - * start postion. - */ - // Calculate the starting position of the specified query statement. - if (*stmt_num == 0) { - query_string_location_start = 0; + for (i = 0; i < len; i++) { + char ch = text[i]; + + if (ch >= 'A' && ch <= 'Z') { + ch += 'a' - 'A'; + } + word[i] = ch; + } + word[len] = '\0'; // The converted text should ends with '\0'. + + /* + * Now do a binary search using plain strcmp() comparison. + */ + low = keywords; + high = keywords + (num_keywords - 1); + while (low <= high) { + const ScanKeyword* middle = NULL; + int difference; + + middle = low + (high - low) / 2; + difference = strcmp(middle->name, word); + if (difference == 0) { + return middle; + } else if (difference < 0) { + low = middle + 1; } else { - query_string_location_start = list_nth_int(query_string_locationlist, *stmt_num - 1) + 1; - } - // Calculate the ending position of the specified query statement. - query_string_location_end = list_nth_int(query_string_locationlist, (*stmt_num)++); - - /* Malloc memory for each single query string. */ - single_query_string_len = query_string_location_end - query_string_location_start + 1; - query_string_single[*stmt_num - 1] = (char*)palloc0(sizeof(char) * (single_query_string_len + 1)); - - /* Copy the query_string between location_start and location_end to query_string_single. */ - query_string_single_p = query_string_single[*stmt_num - 1]; - while (query_string_location_start <= query_string_location_end) { - *query_string_single_p = *(query_string + query_string_location_start); - query_string_location_start++; - query_string_single_p++; - } - - /* - * If query_string_single is empty query which only contain comments or null strings, - * we will skip it. - */ - if (is_empty_query(query_string_single[*stmt_num - 1])) { - continue; - } else { // The obtained query statement is not a null query, exit the loop, and return this statement. - break; + high = middle - 1; } } - - return query_string_single; + // The binary search fails and returns a null value. + return NULL; } \ No newline at end of file -- 2.34.1 From 2cbf94f194e226a384c103de785bd2883cde25ee Mon Sep 17 00:00:00 2001 From: XL_up <3066352084@qq.com> Date: Mon, 28 Aug 2023 23:00:49 +0800 Subject: [PATCH 6/9] Update parser.cpp --- src/common/backend/parser/parser.cpp | 36 +++++++++++++++------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/src/common/backend/parser/parser.cpp b/src/common/backend/parser/parser.cpp index d91fb17c..d7541e83 100644 --- a/src/common/backend/parser/parser.cpp +++ b/src/common/backend/parser/parser.cpp @@ -9,7 +9,6 @@ * the grammar are "raw" parsetrees that still need to be analyzed by * analyze.c and related files. * - * * Portions Copyright (c) 1996-2012, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * @@ -40,15 +39,15 @@ static void resetCreateFuncFlag() /* * raw_parser - * Given a query in string form, do lexical and grammatical analysis. + * Given a query in string form, do lexical and grammatical analysis. * * Returns a list of raw (un-analyzed) parse trees. */ List* raw_parser(const char* str, List** query_string_locationlist) { - core_yyscan_t yyscanner; - base_yy_extra_type yyextra; - int yyresult; + core_yyscan_t yyscanner; + base_yy_extra_type yyextra; // An intermediate variable that holds the returned syntax tree. + int yyresult; // The calling result of base_yyparse(). /* reset u_sess->parser_cxt.stmt_contains_operator_plus */ resetOperatorPlusFlag(); @@ -58,7 +57,7 @@ List* raw_parser(const char* str, List** query_string_locationlist) /* reset u_sess->parser_cxt.isCreateFuncOrProc */ resetCreateFuncFlag(); - + /* initialize the flex scanner */ yyscanner = scanner_init(str, &yyextra.core_yy_extra, ScanKeywords, NumScanKeywords); @@ -89,6 +88,7 @@ List* raw_parser(const char* str, List** query_string_locationlist) } } + // Returns the generated syntax tree. return yyextra.parsetree; } @@ -127,7 +127,7 @@ List* raw_parser(const char* str, List** query_string_locationlist) * words. Furthermore it's not clear how to do it without re-introducing * scanner backtrack, which would cost more performance than this filter * layer does. - * + * * The filter also provides a convenient place to translate between * the core_YYSTYPE and YYSTYPE representations (which are really the * same thing anyway, but notationally they're different). @@ -489,8 +489,8 @@ int base_yylex(YYSTYPE* lvalp, YYLTYPE* llocp, core_yyscan_t yyscanner) /* * @Description: Check whether its a empty query with only comments and semicolon. - * @Param query_string: the query need check. - * @retrun:true or false. + * @Param[IN] query_string: the query need check. + * @return:the bool value of the check result. */ static bool is_empty_query(char* query_string) { @@ -513,6 +513,7 @@ static bool is_empty_query(char* query_string) end_comment_postion = strstr(query_string, end_comment); query_string = end_comment_postion + 2; while (isspace((unsigned char)*query_string)) { + // Trim the spaces after comments. query_string++; } } @@ -537,11 +538,12 @@ static bool is_empty_query(char* query_string) char** get_next_snippet( char** query_string_single, const char* query_string, List* query_string_locationlist, int* stmt_num) { - int query_string_location_start = 0; - int query_string_location_end = -1; - char* query_string_single_p = NULL; - int single_query_string_len = 0; - + int query_string_location_start = 0; // The starting position of the query statement. + int query_string_location_end = -1; // The ending position of the query statement. + char* query_string_single_p = NULL; // An intermediate variable used to copy the string. + int single_query_string_len = 0; // The length of the query statement. + + // Count the number of query statements. int stmt_count = list_length(query_string_locationlist); /* Malloc memory for single query here just for the first time. */ @@ -558,11 +560,13 @@ char** get_next_snippet( * Notice : The locationlist only store the end postion of each single query but not any * start postion. */ + // Calculate the starting position of the specified query statement. if (*stmt_num == 0) { query_string_location_start = 0; } else { query_string_location_start = list_nth_int(query_string_locationlist, *stmt_num - 1) + 1; } + // Calculate the ending position of the specified query statement. query_string_location_end = list_nth_int(query_string_locationlist, (*stmt_num)++); /* Malloc memory for each single query string. */ @@ -583,10 +587,10 @@ char** get_next_snippet( */ if (is_empty_query(query_string_single[*stmt_num - 1])) { continue; - } else { + } else { // The obtained query statement is not a null query, exit the loop, and return this statement. break; } } return query_string_single; -} +} \ No newline at end of file -- 2.34.1 From 35c1219b43c5e1ebe87a28f1de4fac0f1aa37cb1 Mon Sep 17 00:00:00 2001 From: XL_up <3066352084@qq.com> Date: Mon, 28 Aug 2023 23:01:27 +0800 Subject: [PATCH 7/9] Update parse_oper.cpp --- src/common/backend/parser/parse_oper.cpp | 94 ++++++++++++++---------- 1 file changed, 56 insertions(+), 38 deletions(-) diff --git a/src/common/backend/parser/parse_oper.cpp b/src/common/backend/parser/parse_oper.cpp index cf35bc26..98c16b33 100644 --- a/src/common/backend/parser/parse_oper.cpp +++ b/src/common/backend/parser/parse_oper.cpp @@ -2,6 +2,7 @@ * * parse_oper.cpp * handle operator things for parser + * 处理表达式中的操作符 * * Portions Copyright (c) 1996-2012, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California @@ -45,21 +46,24 @@ */ /* If your search_path is longer than this, sucks to be you ... */ +// search_path的最大长度 #define MAX_CACHED_PATH_LEN 16 typedef struct OprCacheKey { - char oprname[NAMEDATALEN]; + char oprname[NAMEDATALEN]; // operator名称 Oid left_arg; /* Left input OID, or 0 if prefix op */ + // 左侧操作符的OID，如果这是一个前缀操作符则此项为0. Oid right_arg; /* Right input OID, or 0 if postfix op */ - Oid search_path[MAX_CACHED_PATH_LEN]; + // 右侧操作符的OID，如果这是一个后缀操作符则此项为0. + Oid search_path[MAX_CACHED_PATH_LEN]; // 搜索路径 bool use_a_style_coercion; } OprCacheKey; typedef struct OprCacheEntry { /* the hash lookup key MUST BE FIRST */ - OprCacheKey key; - - Oid opr_oid; /* OID of the resolved operator */ + OprCacheKey key; // 哈希查找值必须是第一项 + /* OID of the resolved operator */ + Oid opr_oid; // 解决后的OID } OprCacheEntry; static Oid binary_oper_exact(List* opname, Oid arg1, Oid arg2, bool use_a_style_coercion); @@ -90,12 +94,18 @@ static void make_oper_cache_entry(OprCacheKey* key, Oid opr_oid); Oid LookupOperName(ParseState* pstate, List* opername, Oid oprleft, Oid oprright, bool noError, int location) { Oid result; - + // Search the operator Oid according to the value of OprCacheKey.left_arg.oprname, + // OprCacheKey.left_arg and OprCacheKey.right_arg. + // Returns result if the lookup is successful. result = OpernameGetOprid(opername, oprleft, oprright); if (OidIsValid(result)) return result; /* we don't use op_error here because only an exact match is wanted */ + + // The lookup fails, and passed-in param noError = false, + // Indicates that the caller allows a lookup failure to be treated as an error. + // This if-branch is used to report the error messages. if (!noError) { char oprkind; @@ -166,9 +176,9 @@ void get_sort_group_operators( { TypeCacheEntry* typentry = NULL; int cache_flags; - Oid lt_opr; - Oid eq_opr; - Oid gt_opr; + Oid lt_opr; // Oid of '<' + Oid eq_opr; // Oid of '=' + Oid gt_opr; // Oid of '>' bool hashable = false; /* @@ -183,9 +193,9 @@ void get_sort_group_operators( cache_flags = TYPECACHE_LT_OPR | TYPECACHE_EQ_OPR | TYPECACHE_GT_OPR; typentry = lookup_type_cache(argtype, cache_flags); - lt_opr = typentry->lt_opr; - eq_opr = typentry->eq_opr; - gt_opr = typentry->gt_opr; + lt_opr = typentry->lt_opr; + eq_opr = typentry->eq_opr; + gt_opr = typentry->gt_opr; hashable = OidIsValid(typentry->hash_proc); /* Report errors if needed */ @@ -681,7 +691,8 @@ static void op_error( } /* - * Operator expression construction. + * make_op + * Operator expression construction. * * Transform operator expression ensuring type compatibility. * This is where some type conversion happens. @@ -691,19 +702,25 @@ static void op_error( */ Expr* make_op(ParseState* pstate, List* opname, Node* ltree, Node* rtree, int location, bool inNumeric) { - Oid ltypeId, rtypeId; - Operator tup; - Form_pg_operator opform; - Oid actual_arg_types[2]; - Oid declared_arg_types[2]; - int nargs; - List* args = NIL; - Oid rettype; - OpExpr* result = NULL; + // Enter the operator name and the structure tree of the left and right operators, + // and then construct the expression structure for them. + + Oid ltypeId, rtypeId; // Oid of left operator and right operator + Operator tup; // The Operator structure to get before constructing OpExpr + Form_pg_operator opform; // Related to type conversion between Pg_Operator and Operator + Oid actual_arg_types[2]; // Oid of the actual left, right operator determined by ltree, rtree + Oid declared_arg_types[2]; // Oid declared in Pg + int nargs; // the number of left and right operators(0/1/2). + List* args = NIL; // tree structure representing the left and right operators + Oid rettype; // Oid of the operator type returned + OpExpr* result = NULL; // The expression structure returned /* Select the operator */ + + // Get the OID of the left and right operators, and then obtain + // the Operator structure of the left and right operators accordingly if (rtree == NULL) { - /* right operator */ + /* right operator */ ltypeId = exprType(ltree); rtypeId = InvalidOid; tup = right_oper(pstate, opname, ltypeId, false, location); @@ -724,7 +741,7 @@ Expr* make_op(ParseState* pstate, List* opname, Node* ltree, Node* rtree, int lo } tup = oper(pstate, opname, ltypeId, rtypeId, false, location, inNumeric); } - + // Related to type conversion between Pg_Operator and Operator opform = check_operator_is_shell(opname, pstate, location, tup); /* Do typecasting and build the expression tree */ @@ -732,13 +749,13 @@ Expr* make_op(ParseState* pstate, List* opname, Node* ltree, Node* rtree, int lo /* right operator */ args = list_make1(ltree); actual_arg_types[0] = ltypeId; - declared_arg_types[0] = opform->oprleft; - nargs = 1; + declared_arg_types[0] = opform->oprleft; + nargs = 1; } else if (ltree == NULL) { /* left operator */ - args = list_make1(rtree); - actual_arg_types[0] = rtypeId; - declared_arg_types[0] = opform->oprright; + args = list_make1(rtree); + actual_arg_types[0] = rtypeId; + declared_arg_types[0] = opform->oprright; nargs = 1; } else { /* otherwise, binary operator */ @@ -747,7 +764,7 @@ Expr* make_op(ParseState* pstate, List* opname, Node* ltree, Node* rtree, int lo actual_arg_types[1] = rtypeId; declared_arg_types[0] = opform->oprleft; declared_arg_types[1] = opform->oprright; - nargs = 2; + nargs = 2; // Both the left operator and the right operator are valid. } /* @@ -755,6 +772,7 @@ Expr* make_op(ParseState* pstate, List* opname, Node* ltree, Node* rtree, int lo * possibly adjusting return type or declared_arg_types (which will be * used as the cast destination by make_fn_arguments) */ + // Oid of the operator type returned rettype = enforce_generic_type_consistency(actual_arg_types, declared_arg_types, nargs, opform->oprresult, false); /* perform the necessary typecasting of arguments */ @@ -762,17 +780,17 @@ Expr* make_op(ParseState* pstate, List* opname, Node* ltree, Node* rtree, int lo /* and build the expression node */ result = makeNode(OpExpr); - result->opno = oprid(tup); - result->opfuncid = opform->oprcode; - result->opresulttype = rettype; + result->opno = oprid(tup); + result->opfuncid = opform->oprcode; + result->opresulttype = rettype; result->opretset = get_func_retset(opform->oprcode); /* opcollid and inputcollid will be set by parse_collate.c */ - result->args = args; - result->location = location; + result->args = args; + result->location = location; - ReleaseSysCache(tup); + ReleaseSysCache(tup); // Release the cache space - return (Expr*)result; + return (Expr*)result; // Returned as an Expr* type } /* @@ -1026,4 +1044,4 @@ void InvalidateOprCacheCallBack(Datum arg, int cacheid, uint32 hashvalue) if (hash_search(u_sess->parser_cxt.opr_cache_hash, (void*)&hentry->key, HASH_REMOVE, NULL) == NULL) ereport(ERROR, (errcode(ERRCODE_DATA_EXCEPTION), errmsg("hash table corrupted"))); } -} +} \ No newline at end of file -- 2.34.1 From 6111027ed4bf1d3efe74725ca4ec920f7d6b9c06 Mon Sep 17 00:00:00 2001 From: XL_up <3066352084@qq.com> Date: Mon, 28 Aug 2023 23:01:48 +0800 Subject: [PATCH 8/9] Update parse_param.cpp --- src/common/backend/parser/parse_param.cpp | 24 ++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/common/backend/parser/parse_param.cpp b/src/common/backend/parser/parse_param.cpp index 661ec4a8..c9de7007 100644 --- a/src/common/backend/parser/parse_param.cpp +++ b/src/common/backend/parser/parse_param.cpp @@ -3,7 +3,7 @@ * parse_param.cpp * handle parameters in parser * - * This code covers two cases that are used within the core backend: + * This code covers two cases that are used within the core backend: * * a fixed list of parameters with known types * * an expandable list of parameters whose types can optionally * be determined from context @@ -33,7 +33,8 @@ #include "utils/builtins.h" #include "utils/lsyscache.h" -typedef struct FixedParamState { +// Stmt of fixed parameters. +typedef struct FixedParamState { Oid* paramTypes; /* array of parameter type OIDs */ int numParams; /* number of array entries */ } FixedParamState; @@ -44,6 +45,7 @@ typedef struct FixedParamState { * hasn't been seen, while UNKNOWNOID means the parameter has been used but * its type is not yet known. */ +// Stmt of variable parameters. typedef struct VarParamState { Oid** paramTypes; /* array of parameter type OIDs */ int* numParams; /* number of array entries */ @@ -64,10 +66,12 @@ static bool check_parameter_resolution_walker(Node* node, ParseState* pstate); */ void parse_fixed_parameters(ParseState* pstate, Oid* paramTypes, int numParams) { + // palloc for FixedParamState. FixedParamState* parstate = (FixedParamState*)palloc(sizeof(FixedParamState)); - + // Construct the FixedParamState structure. parstate->paramTypes = paramTypes; parstate->numParams = numParams; + pstate->p_ref_hook_state = (void*)parstate; pstate->p_paramref_hook = fixed_paramref_hook; /* no need to use p_coerce_param_hook */ @@ -78,10 +82,12 @@ void parse_fixed_parameters(ParseState* pstate, Oid* paramTypes, int numParams) */ void parse_variable_parameters(ParseState* pstate, Oid** paramTypes, int* numParams) { + // palloc for VarParamState VarParamState* parstate = (VarParamState*)palloc(sizeof(VarParamState)); - + // Construct the VarParamState structure. parstate->paramTypes = paramTypes; parstate->numParams = numParams; + pstate->p_ref_hook_state = (void*)parstate; pstate->p_paramref_hook = variable_paramref_hook; pstate->p_coerce_param_hook = variable_coerce_param_hook; @@ -265,10 +271,10 @@ static bool check_parameter_resolution_walker(Node* node, ParseState* pstate) if (node == NULL) { return false; } - if (IsA(node, Param)) { - Param* param = (Param*)node; + if (IsA(node, Param)) { // If the current node is an object of the Param class + Param* param = (Param*)node; - if (param->paramkind == PARAM_EXTERN) { + if (param->paramkind == PARAM_EXTERN) { // If param is external VarParamState* parstate = (VarParamState*)pstate->p_ref_hook_state; int paramno = param->paramid; @@ -288,9 +294,9 @@ static bool check_parameter_resolution_walker(Node* node, ParseState* pstate) } return false; } - if (IsA(node, Query)) { + if (IsA(node, Query)) { // If the current node is an object of the Query class /* Recurse into RTE subquery or not-yet-planned sublink subquery */ return query_tree_walker((Query*)node, (bool (*)())check_parameter_resolution_walker, (void*)pstate, 0); } return expression_tree_walker(node, (bool (*)())check_parameter_resolution_walker, (void*)pstate); -} +} \ No newline at end of file -- 2.34.1 From 468892b8174682ce84c55dbab38b9976106a3de9 Mon Sep 17 00:00:00 2001 From: XL_up <3066352084@qq.com> Date: Mon, 28 Aug 2023 23:02:20 +0800 Subject: [PATCH 9/9] Update analyze.cpp --- src/common/backend/parser/analyze.cpp | 36 +++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/src/common/backend/parser/analyze.cpp b/src/common/backend/parser/analyze.cpp index 2baa4184..3f64e808 100644 --- a/src/common/backend/parser/analyze.cpp +++ b/src/common/backend/parser/analyze.cpp @@ -152,31 +152,55 @@ static const char* NOKEYUPDATE_KEYSHARE_ERRMSG = ""; * transformation, while utility-type statements are simply hung off * a dummy CMD_UTILITY Query node. */ + + /* + * @Description：Analyze the raw syntax tree, do semantic analysis and output query tree. + * @Param[IN] parseTree: a raw parse tree(abstract syntax tree). + * @Param[IN] sourceText: The source text of the query statement. + * @Param[IN] numParams: number of params. + * @Param[IN] paramTypes: stores the type OID of each parameter. + * @Return：an analyzed query tree(of Query class). + */ Query* parse_analyze( Node* parseTree, const char* sourceText, Oid* paramTypes, int numParams, bool isFirstNode, bool isCreateView) { + // Initialization of ParseState object. ParseState* pstate = make_parsestate(NULL); + // Declaration of the query tree. Query* query = NULL; - /* required as of 8.4 */ + // sourceText can't be NULL. AssertEreport(sourceText != NULL, MOD_OPT, "para cannot be NULL"); pstate->p_sourcetext = sourceText; if (numParams > 0) { + /* + * parse_fixed_parameters() was defined in parse_param.cpp. + * + * This function converts the reference contained in the query into a + * fixed parameter structure, that is, pass in the reference parameters + * of the query statement and constructs a FixedParamState structure + * for it. + */ parse_fixed_parameters(pstate, paramTypes, numParams); } PUSH_SKIP_UNIQUE_SQL_HOOK(); + + // Converts the raw parse tree into a query tree. query = transformTopLevelStmt(pstate, parseTree, isFirstNode, isCreateView); - POP_SKIP_UNIQUE_SQL_HOOK(); - + + POP_SKIP_UNIQUE_SQL_HOOK(); + /* it's unsafe to deal with plugins hooks as dynamic lib may be released */ if (post_parse_analyze_hook && !(g_instance.status > NoShutdown)) { (*post_parse_analyze_hook)(pstate, query); } pfree_ext(pstate->p_ref_hook_state); + + // release the ParseState object pstate. free_parsestate(pstate); /* For plpy CTAS query. CTAS is a recursive call. CREATE query is the first rewrited. @@ -185,7 +209,8 @@ Query* parse_analyze( */ query->fixed_paramTypes = paramTypes; query->fixed_numParams = numParams; - + + // return the analyzed raw parse tree, that is, the query tree. return query; } @@ -228,6 +253,7 @@ Query* parse_analyze_varparams(Node* parseTree, const char* sourceText, Oid** pa /* * parse_sub_analyze * Entry point for recursively analyzing a sub-statement. + * 递归分析子查询的入口函数 */ Query* parse_sub_analyze(Node* parseTree, ParseState* parentParseState, CommonTableExpr* parentCTE, bool locked_from_parent, bool resolve_unknowns) @@ -4732,4 +4758,4 @@ static bool checkAllowedTableCombination(ParseState* pstate) Assert(has_ustore || has_else); return !(has_ustore && has_else); -} +} \ No newline at end of file -- 2.34.1