HCK-14321: Update Grammar & Visitor to Parse Streaming Tables

reverse_engineering/commandsService.js

@@ -149,22 +149,36 @@ const convertCommandsToReDocs = (commands, originalScript) => {
 	return { result, info: reData.modelProperties, relationships: reData.relationships };
 };
 
-const createCollection = (entitiesData, bucket, statementData) => {
+const createCollection = (entitiesData, bucket, statementData, originalScript) => {
 	const { entities, currentBucket } = entitiesData;
 	const updatedEntityData = getTableMergedWithReferencedTable(entities, statementData);
+	const streamingSourceSelect = cleanUpSelectStatement(
+		originalScript.substring(statementData.select.start, statementData.select.stop),
+	);
+
+	const entityData = {
+		...updatedEntityData,
+		entityLevelData: {
+			...updatedEntityData.entityLevelData,
+			streamingSourceSelect,
+		},
+	};
 
 	if (!updatedEntityData.bucketName) {
-		return { ...entitiesData, entities: [...entities, { ...updatedEntityData, bucketName: bucket }] };
+		return {
+			...entitiesData,
+			entities: [...entities, { ...entityData, bucketName: bucket }],
+		};
 	}
 
 	if (currentBucket === DEFAULT_BUCKET) {
 		return {
 			...entitiesData,
-			entities: [...entities, updatedEntityData],
+			entities: [...entities, entityData],
 			bucketName: updatedEntityData.bucketName,
 		};
 	} else {
-		return { ...entitiesData, entities: [...entities, updatedEntityData] };
+		return { ...entitiesData, entities: [...entities, entityData] };
 	}
 };
 

reverse_engineering/grammars/FromClauseParser.g4

@@ -49,6 +49,7 @@ fromSource
     : uniqueJoinToken uniqueJoinSource (COMMA uniqueJoinSource)+
     | joinSource
     | rangeTableValuedFunction
+    | streamSource
     ;
 
 
@@ -64,6 +65,11 @@ joinSource
     : atomjoinSource (joinToken joinSourcePart (KW_ON expression | KW_USING columnParenthesesList)?)*
     ;
 
+streamSource
+    : KW_STREAM LPAREN? tableName RPAREN? tableAlias?
+    | KW_STREAM tableFunctionCall
+    ;
+
 joinSourcePart
     : (tableSource | virtualTableSource | subQuerySource | partitionedTableFunction) lateralView*
     ;
@@ -201,3 +207,11 @@ rangeTableValuedFunction
     ;
 
 //-----------------------------------------------------------------------------------
+
+tableFunctionCall: identifier LPAREN functionArgList? RPAREN;
+
+functionArgList: functionArg (',' functionArg)*;
+
+functionArg: namedArg | expression | StringLiteral;
+
+namedArg: identifier FAT_ARROW (identifier | expression | StringLiteral);

reverse_engineering/grammars/HiveLexer.g4

@@ -381,6 +381,12 @@ KW_EVERY: E V E R Y;
 KW_CRON: C R O N;
 KW_AT: A T;
 KW_FILTER: F I L T E R;
+KW_STREAM: S T R E A M;
+KW_STREAMING: S T R E A M I N G;
+KW_MOST: M O S T;
+KW_EXPECT: E X P E C T;
+KW_VIOLATION: V I O L A T I O N;
+KW_FAIL: F A I L;
 // Operators
 // NOTE: if you add a new function/operator, add it to sysFuncNames so that describe function _FUNC_ will work.
 
@@ -403,6 +409,7 @@ LESSTHANOREQUALTO : '<=';
 LESSTHAN : '<';
 GREATERTHANOREQUALTO : '>=';
 GREATERTHAN : '>';
+FAT_ARROW: '=>';
 
 DIVIDE : '/';
 PLUS : '+';

reverse_engineering/grammars/HiveParser.g4

@@ -217,6 +217,10 @@ orReplace
     : KW_OR KW_REPLACE
     ;
 
+orRefresh
+    : KW_OR KW_REFRESH
+    ;
+
 createDatabaseStatement
     : KW_CREATE (KW_DATABASE|KW_SCHEMA)
         ifNotExists?
@@ -258,7 +262,7 @@ databaseComment
     ;
 
 createTableStatement
-    : KW_CREATE KW_TEMPORARY? KW_EXTERNAL? KW_TABLE ifNotExists? tableName
+    : KW_CREATE orRefresh? KW_TEMPORARY? KW_EXTERNAL? KW_STREAMING? KW_TABLE ifNotExists? tableName
       (  KW_LIKE tableName
          tableRowFormat?
          tableFileFormat?
@@ -278,6 +282,9 @@ createTableStatement
          | tableOptions
          | tableComment
          | clusterByClause
+         | scheduleClause
+         | rowClause
+         | triggerOnUpdateClause
         )*
          (KW_AS selectStatementWithCTE)?
       )
@@ -905,13 +912,30 @@ materializedViewClause
 
 scheduleClause
     : KW_SCHEDULE KW_REFRESH? KW_EVERY Number (KW_HOUR | KW_DAY | KW_WEEK)
-    | KW_SCHEDULE KW_REFRESH? KW_CRON Identifier (KW_AT KW_TIME KW_ZONE Identifier)?
+    | KW_SCHEDULE KW_REFRESH? KW_CRON identifier (KW_AT KW_TIME KW_ZONE identifier)?
     ;
 
 rowClause
     : KW_WITH? KW_ROW KW_FILTER functionIdentifier KW_ON (LPAREN identifier (COMMA identifier)* RPAREN)?
     ;
 
+triggerOnUpdateClause
+    : KW_TRIGGER KW_ON KW_UPDATE (KW_AT KW_MOST KW_EVERY intervalClause)?
+    ;
+
+intervalClause
+    : KW_INTERVAL Number? intervalQualifier
+    ;
+
+intervalQualifier
+    : KW_YEAR (KW_TO KW_MONTH)?
+    | KW_MONTH
+    | KW_DAY (KW_TO (KW_HOUR | KW_MINUTE | KW_SECOND))?
+    | KW_HOUR (KW_TO (KW_MINUTE | KW_SECOND))?
+    | KW_MINUTE (KW_TO KW_SECOND)?
+    | KW_SECOND
+    ;
+
 viewPartition
     : KW_PARTITIONED KW_ON LPAREN columnNameList RPAREN
     ;
@@ -1152,6 +1176,8 @@ alterConstraintWithName
 tableLevelConstraint
     : pkUkConstraint
     | checkConstraint
+    | expectConstraint
+    | createForeignKey
     ;
 
 pkUkConstraint
@@ -1162,6 +1188,10 @@ checkConstraint
     : KW_CHECK expression
     ;
 
+expectConstraint
+    : KW_EXPECT LPAREN expression RPAREN (KW_ON KW_VIOLATION (KW_FAIL KW_UPDATE | KW_DROP KW_ROW))?
+    ;
+
 createForeignKey
     : (KW_CONSTRAINT identifier)? KW_FOREIGN KW_KEY columnParenthesesList  KW_REFERENCES tableName columnParenthesesList constraintOptsCreate?
     ;
@@ -1237,7 +1267,7 @@ tableConstraint
     ;
 
 columnNameTypeConstraint
-    : identifier colType columnConstraint? (KW_COMMENT StringLiteral)? (KW_MASK functionIdentifier)?
+    : identifier colType columnConstraint*
     ;
 
 columnGeneratedAs
@@ -1300,6 +1330,8 @@ columnConstraintType
     | columnGeneratedAs
     | checkConstraint
     | tableConstraintType
+    | KW_COMMENT StringLiteral
+    | KW_MASK functionIdentifier
     ;
 
 defaultVal

reverse_engineering/grammars/SelectClauseParser.g4

@@ -39,7 +39,7 @@ selectTrfmClause
     ;
 
 selectItem
-    : tableAllColumns
+    : tableAllColumns exceptClause?
     | ( expression
       ((KW_AS? identifier) | (KW_AS LPAREN identifier (COMMA identifier)* RPAREN))?
     )
@@ -55,14 +55,18 @@ trfmClause
     ;
 
 selectExpression
-    : tableAllColumns
+    : tableAllColumns exceptClause?
     | expression
     ;
 
 selectExpressionList
     : selectExpression (COMMA selectExpression)*
     ;
 
+exceptClause
+    : KW_EXCEPT LPAREN selectExpressionList RPAREN
+    ;
+
 //---------------------- Rules for windowing clauses -------------------------------
 window_clause
     :