diff --git a/docs/content/docs/sql/reference/data-types.md b/docs/content/docs/sql/reference/data-types.md index 45ace31e365083..eb5eface7ba840 100644 --- a/docs/content/docs/sql/reference/data-types.md +++ b/docs/content/docs/sql/reference/data-types.md @@ -219,6 +219,7 @@ The default planner supports the following set of SQL types: | Structured types | Only exposed in user-defined functions yet. | | `VARIANT` | | | `BITMAP` | | +| `GEOGRAPHY` | Geography values in OGC:CRS84. | ### Character Strings @@ -1597,6 +1598,47 @@ DataTypes.BITMAP() {{< /tab >}} {{< /tabs >}} +#### `GEOGRAPHY` + +Data type of geography data. + +`GEOGRAPHY` represents geospatial values in the OGC:CRS84 coordinate reference system. +The type itself does not define SQL constructors, accessors, or spatial predicate functions. +Those functions are expected to be added separately. + +Flink represents geography payloads as ISO WKB bytes. ISO WKB does not encode CRS or SRID +metadata, so CRS validation, CRS transformation, and EWKB/SRID handling belong to +constructors, functions, or connector-specific schema mapping. + +The geography type is an extension to the SQL standard. + +**Declaration** + +{{< tabs "9fef5895-3fa0-4b9b-9f92-9c94ba96ef1a" >}} +{{< tab "SQL" >}} +```text +GEOGRAPHY +``` + +{{< /tab >}} +{{< tab "Java/Scala" >}} +```java +DataTypes.GEOGRAPHY() +``` + +**Bridging to JVM Types** + +| Java Type | Input | Output | Remarks | +|:-----------------------------------------------|:-----:|:------:|:----------| +| `org.apache.flink.table.data.GeographyData` | X | X | *Default* | + +{{< /tab >}} +{{< /tabs >}} + +`GEOGRAPHY` values cannot be constructed with `CAST` from character or binary string +types. Likewise, `GEOGRAPHY` values cannot be cast to character or binary string types. +Use explicit geography functions for those conversions once such functions are available. + #### `RAW` Data type of an arbitrary serialized type. This type is a black box within the table ecosystem diff --git a/docs/static/generated/rest_v1_sql_gateway.yml b/docs/static/generated/rest_v1_sql_gateway.yml index fecf0576a011cd..baee191c2a042d 100644 --- a/docs/static/generated/rest_v1_sql_gateway.yml +++ b/docs/static/generated/rest_v1_sql_gateway.yml @@ -403,6 +403,7 @@ components: - DESCRIPTOR - VARIANT - BITMAP + - GEOGRAPHY OpenSessionRequestBody: type: object properties: diff --git a/docs/static/generated/rest_v2_sql_gateway.yml b/docs/static/generated/rest_v2_sql_gateway.yml index e39feaaea8ed53..0b6b70219283b4 100644 --- a/docs/static/generated/rest_v2_sql_gateway.yml +++ b/docs/static/generated/rest_v2_sql_gateway.yml @@ -477,6 +477,7 @@ components: - DESCRIPTOR - VARIANT - BITMAP + - GEOGRAPHY OpenSessionRequestBody: type: object properties: diff --git a/docs/static/generated/rest_v3_sql_gateway.yml b/docs/static/generated/rest_v3_sql_gateway.yml index 6d171ac2152f37..bc46614b4248b6 100644 --- a/docs/static/generated/rest_v3_sql_gateway.yml +++ b/docs/static/generated/rest_v3_sql_gateway.yml @@ -506,6 +506,7 @@ components: - DESCRIPTOR - VARIANT - BITMAP + - GEOGRAPHY OpenSessionRequestBody: type: object properties: diff --git a/docs/static/generated/rest_v4_sql_gateway.yml b/docs/static/generated/rest_v4_sql_gateway.yml index d39da5e007c2b7..b1d917452af648 100644 --- a/docs/static/generated/rest_v4_sql_gateway.yml +++ b/docs/static/generated/rest_v4_sql_gateway.yml @@ -516,6 +516,7 @@ components: - DESCRIPTOR - VARIANT - BITMAP + - GEOGRAPHY OpenSessionRequestBody: type: object properties: diff --git a/flink-formats/flink-parquet/pom.xml b/flink-formats/flink-parquet/pom.xml index 09e9bb316ff1fb..a506ed4c938cc6 100644 --- a/flink-formats/flink-parquet/pom.xml +++ b/flink-formats/flink-parquet/pom.xml @@ -80,6 +80,14 @@ under the License. true + + org.apache.flink + flink-connector-base + ${project.version} + provided + true + + org.apache.flink flink-hadoop-fs diff --git a/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/row/ParquetRowDataBuilder.java b/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/row/ParquetRowDataBuilder.java index 5abb21d5a269c1..2afc44a132ee7b 100644 --- a/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/row/ParquetRowDataBuilder.java +++ b/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/row/ParquetRowDataBuilder.java @@ -21,6 +21,7 @@ import org.apache.flink.api.common.serialization.BulkWriter; import org.apache.flink.formats.parquet.ParquetBuilder; import org.apache.flink.formats.parquet.ParquetWriterFactory; +import org.apache.flink.formats.parquet.utils.GeoParquetMetadataUtil; import org.apache.flink.formats.parquet.utils.SerializableConfiguration; import org.apache.flink.table.data.RowData; import org.apache.flink.table.types.logical.RowType; @@ -35,7 +36,7 @@ import org.apache.parquet.schema.MessageType; import java.io.IOException; -import java.util.HashMap; +import java.util.Map; import static org.apache.flink.formats.parquet.utils.ParquetSchemaConverter.convertToParquetMessageType; import static org.apache.parquet.hadoop.ParquetOutputFormat.MAX_PADDING_BYTES; @@ -81,7 +82,9 @@ private ParquetWriteSupport(Configuration conf) { @Override public WriteContext init(Configuration configuration) { - return new WriteContext(schema, new HashMap<>()); + final Map keyValueMetaData = + GeoParquetMetadataUtil.createGeoParquetKeyValueMetaData(rowType); + return new WriteContext(schema, keyValueMetaData); } @Override diff --git a/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/row/ParquetRowDataWriter.java b/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/row/ParquetRowDataWriter.java index 28ae2d8b62a662..0b9616eef1dbdc 100644 --- a/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/row/ParquetRowDataWriter.java +++ b/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/row/ParquetRowDataWriter.java @@ -119,6 +119,8 @@ private FieldWriter createWriter(LogicalType t, Type type) { case BINARY: case VARBINARY: return new BinaryWriter(); + case GEOGRAPHY: + return new GeographyWriter(); case DECIMAL: DecimalType decimalType = (DecimalType) t; return createDecimalWriter(decimalType.getPrecision(), decimalType.getScale()); @@ -311,6 +313,23 @@ private void writeBinary(byte[] value) { } } + private class GeographyWriter implements FieldWriter { + + @Override + public void write(RowData row, int ordinal) { + writeGeography(row.getGeography(ordinal).toBytes()); + } + + @Override + public void write(ArrayData arrayData, int ordinal) { + writeGeography(arrayData.getGeography(ordinal).toBytes()); + } + + private void writeGeography(byte[] value) { + recordConsumer.addBinary(Binary.fromReusedByteArray(value)); + } + } + private class IntWriter implements FieldWriter { @Override diff --git a/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/utils/GeoParquetMetadataUtil.java b/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/utils/GeoParquetMetadataUtil.java new file mode 100644 index 00000000000000..8da802292c48f0 --- /dev/null +++ b/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/utils/GeoParquetMetadataUtil.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.parquet.utils; + +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.LogicalTypeRoot; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.util.jackson.JacksonMapperFactory; + +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; + +import org.apache.parquet.schema.MessageType; + +import java.io.IOException; +import java.util.Collections; +import java.util.Map; + +/** Utilities for emitting and validating GeoParquet metadata. */ +public final class GeoParquetMetadataUtil { + + public static final String GEO_METADATA_KEY = "geo"; + public static final String GEOPARQUET_VERSION = "1.1.0"; + + private static final String GEOGRAPHY_ENCODING = "WKB"; + private static final String GEOGRAPHY_EDGES = "spherical"; + private static final ObjectMapper OBJECT_MAPPER = JacksonMapperFactory.createObjectMapper(); + + private GeoParquetMetadataUtil() {} + + public static Map createGeoParquetKeyValueMetaData(RowType rowType) { + final ObjectNode columnsNode = OBJECT_MAPPER.createObjectNode(); + String primaryColumn = null; + + for (int i = 0; i < rowType.getFieldCount(); i++) { + final LogicalType fieldType = rowType.getTypeAt(i); + if (fieldType.getTypeRoot() != LogicalTypeRoot.GEOGRAPHY) { + continue; + } + + final String fieldName = rowType.getFieldNames().get(i); + if (primaryColumn == null) { + primaryColumn = fieldName; + } + + final ObjectNode columnNode = columnsNode.putObject(fieldName); + columnNode.put("encoding", GEOGRAPHY_ENCODING); + columnNode.putArray("geometry_types"); + columnNode.put("edges", GEOGRAPHY_EDGES); + } + + if (primaryColumn == null) { + return Collections.emptyMap(); + } + + final ObjectNode rootNode = OBJECT_MAPPER.createObjectNode(); + rootNode.put("version", GEOPARQUET_VERSION); + rootNode.put("primary_column", primaryColumn); + rootNode.set("columns", columnsNode); + + try { + return Collections.singletonMap( + GEO_METADATA_KEY, OBJECT_MAPPER.writeValueAsString(rootNode)); + } catch (JsonProcessingException e) { + throw new IllegalStateException("Failed to serialize GeoParquet metadata.", e); + } + } + + public static void validateGeoParquetMetadata( + Map keyValueMetaData, + MessageType requestedSchema, + LogicalType[] selectedTypes) + throws IOException { + if (keyValueMetaData == null || !keyValueMetaData.containsKey(GEO_METADATA_KEY)) { + return; + } + + final JsonNode geoNode = OBJECT_MAPPER.readTree(keyValueMetaData.get(GEO_METADATA_KEY)); + final JsonNode columnsNode = geoNode.get("columns"); + if (columnsNode == null || !columnsNode.isObject()) { + throw new IOException("Invalid GeoParquet metadata: missing 'columns' object."); + } + + for (int i = 0; i < selectedTypes.length; i++) { + if (selectedTypes[i].getTypeRoot() != LogicalTypeRoot.GEOGRAPHY) { + continue; + } + + final String columnName = requestedSchema.getFields().get(i).getName(); + final JsonNode columnNode = columnsNode.get(columnName); + if (columnNode == null || !columnNode.isObject()) { + throw new IOException( + String.format( + "Invalid GeoParquet metadata: missing GEOGRAPHY column '%s'.", + columnName)); + } + + final JsonNode encodingNode = columnNode.get("encoding"); + if (encodingNode == null || !encodingNode.isTextual()) { + throw new IOException( + String.format( + "Invalid GeoParquet metadata: missing encoding for GEOGRAPHY column '%s'.", + columnName)); + } + + if (!GEOGRAPHY_ENCODING.equals(encodingNode.asText())) { + throw new IOException( + String.format( + "Unsupported GeoParquet encoding '%s' for GEOGRAPHY column '%s'.", + encodingNode.asText(), columnName)); + } + } + } +} diff --git a/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/utils/ParquetFormatStatisticsReportUtil.java b/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/utils/ParquetFormatStatisticsReportUtil.java index 9eed138bab7488..9c338094996605 100644 --- a/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/utils/ParquetFormatStatisticsReportUtil.java +++ b/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/utils/ParquetFormatStatisticsReportUtil.java @@ -61,7 +61,6 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; -import java.util.stream.Collectors; import static org.apache.flink.util.Preconditions.checkArgument; @@ -103,20 +102,22 @@ public static TableStats getTableStatistics( for (Path file : files) { fileRowCountFutures.add( executorService.submit( - new ParquetFileRowCountCalculator( - hadoopConfig, file, columnStatisticsMap))); + new ParquetFileRowCountCalculator(hadoopConfig, file))); } for (Future fileCountFuture : fileRowCountFutures) { FileParquetStatistics fileStatistics = fileCountFuture.get(); - List columns = fileStatistics.getColumns(); - List blocks = fileStatistics.blocks; + List columns = fileStatistics.getLeafColumnNames(); + List blocks = fileStatistics.getBlocks(); for (BlockMetaData block : blocks) { rowCount += block.getRowCount(); - for (int i = 0; i < columns.size(); ++i) { - updateStatistics( - block.getColumns().get(i).getStatistics(), - columns.get(i), - columnStatisticsMap); + for (int i = 0; i < Math.min(columns.size(), block.getColumns().size()); ++i) { + String column = columns.get(i); + if (column != null) { + updateStatistics( + block.getColumns().get(i).getStatistics(), + column, + columnStatisticsMap); + } } } } @@ -151,20 +152,33 @@ private static Map convertToColumnStats( boolean isUtcTimestamp) { Map columnStatMap = new HashMap<>(); for (String column : producedRowType.getFieldNames()) { + LogicalType logicalType = + producedRowType.getTypeAt(producedRowType.getFieldIndex(column)); Statistics statistics = columnStatisticsMap.get(column); if (statistics == null) { + if (shouldReportNullColumnStats(logicalType)) { + columnStatMap.put(column, null); + } continue; } - ColumnStats columnStats = - convertToColumnStats( - producedRowType.getTypeAt(producedRowType.getFieldIndex(column)), - statistics, - isUtcTimestamp); + ColumnStats columnStats = convertToColumnStats(logicalType, statistics, isUtcTimestamp); columnStatMap.put(column, columnStats); } return columnStatMap; } + private static boolean shouldReportNullColumnStats(LogicalType logicalType) { + switch (logicalType.getTypeRoot()) { + case ARRAY: + case MAP: + case MULTISET: + case ROW: + return true; + default: + return false; + } + } + private static ColumnStats convertToColumnStats( LogicalType logicalType, Statistics statistics, boolean isUtcTimestamp) { ColumnStats.Builder builder = @@ -176,6 +190,7 @@ private static ColumnStats convertToColumnStats( case BOOLEAN: case BINARY: case VARBINARY: + case GEOGRAPHY: break; case TINYINT: case SMALLINT: @@ -328,19 +343,39 @@ private static Timestamp binaryToTimestamp(Binary timestamp, boolean utcTimestam return timestampData.toTimestamp(); } + private static List getLeafColumnNames(MessageType schema) { + List leafColumnNames = new ArrayList<>(); + for (Type field : schema.getFields()) { + addLeafColumnNames(field, field.getName(), leafColumnNames); + } + return leafColumnNames; + } + + private static void addLeafColumnNames( + Type field, String topLevelFieldName, List leafColumnNames) { + if (field.isPrimitive()) { + leafColumnNames.add(topLevelFieldName); + return; + } + + for (Type nestedField : field.asGroupType().getFields()) { + addLeafColumnNames(nestedField, null, leafColumnNames); + } + } + private static class FileParquetStatistics { - private final List columns; + private final List leafColumnNames; private final List blocks; - public FileParquetStatistics(List columns, List blocks) { - this.columns = columns; + public FileParquetStatistics(List leafColumnNames, List blocks) { + this.leafColumnNames = leafColumnNames; this.blocks = blocks; } - public List getColumns() { - return columns; + public List getLeafColumnNames() { + return leafColumnNames; } public List getBlocks() { @@ -352,10 +387,7 @@ private static class ParquetFileRowCountCalculator implements Callable> columnStatisticsMap) { + public ParquetFileRowCountCalculator(Configuration hadoopConfig, Path file) { this.hadoopConfig = hadoopConfig; this.file = file; } @@ -365,12 +397,8 @@ public FileParquetStatistics call() throws Exception { org.apache.hadoop.fs.Path hadoopPath = new org.apache.hadoop.fs.Path(file.toUri()); ParquetMetadata metadata = ParquetFileReader.readFooter(hadoopConfig, hadoopPath); MessageType schema = metadata.getFileMetaData().getSchema(); - List columns = - schema.asGroupType().getFields().stream() - .map(Type::getName) - .collect(Collectors.toList()); List blocks = metadata.getBlocks(); - return new FileParquetStatistics(columns, blocks); + return new FileParquetStatistics(getLeafColumnNames(schema), blocks); } } } diff --git a/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/utils/ParquetSchemaConverter.java b/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/utils/ParquetSchemaConverter.java index 0101fc13f428e3..4e454e2f8c8979 100644 --- a/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/utils/ParquetSchemaConverter.java +++ b/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/utils/ParquetSchemaConverter.java @@ -81,6 +81,9 @@ private static Type convertToParquetType( case VARBINARY: return Types.primitive(PrimitiveType.PrimitiveTypeName.BINARY, repetition) .named(name); + case GEOGRAPHY: + return Types.primitive(PrimitiveType.PrimitiveTypeName.BINARY, repetition) + .named(name); case DECIMAL: int precision = ((DecimalType) type).getPrecision(); int scale = ((DecimalType) type).getScale(); diff --git a/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/vector/ParquetColumnarRowSplitReader.java b/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/vector/ParquetColumnarRowSplitReader.java index 7634978e916406..f4d0e268205577 100644 --- a/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/vector/ParquetColumnarRowSplitReader.java +++ b/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/vector/ParquetColumnarRowSplitReader.java @@ -18,6 +18,9 @@ package org.apache.flink.formats.parquet.vector; +import org.apache.flink.core.fs.FileSystem; +import org.apache.flink.formats.parquet.ParquetInputFile; +import org.apache.flink.formats.parquet.utils.GeoParquetMetadataUtil; import org.apache.flink.formats.parquet.vector.reader.ColumnReader; import org.apache.flink.formats.parquet.vector.type.ParquetField; import org.apache.flink.table.data.columnar.ColumnarRowData; @@ -31,12 +34,12 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.parquet.ParquetReadOptions; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.column.page.PageReadStore; import org.apache.parquet.filter2.compat.FilterCompat; import org.apache.parquet.hadoop.ParquetFileReader; import org.apache.parquet.hadoop.metadata.BlockMetaData; -import org.apache.parquet.hadoop.metadata.ParquetMetadata; import org.apache.parquet.io.ColumnIOFactory; import org.apache.parquet.io.MessageColumnIO; import org.apache.parquet.schema.GroupType; @@ -55,9 +58,6 @@ import static org.apache.flink.formats.parquet.vector.ParquetSplitReaderUtil.buildFieldsList; import static org.apache.flink.formats.parquet.vector.ParquetSplitReaderUtil.createColumnReader; import static org.apache.flink.formats.parquet.vector.ParquetSplitReaderUtil.createWritableColumnVector; -import static org.apache.parquet.filter2.compat.RowGroupFilter.filterRowGroups; -import static org.apache.parquet.format.converter.ParquetMetadataConverter.range; -import static org.apache.parquet.hadoop.ParquetFileReader.readFooter; import static org.apache.parquet.hadoop.ParquetInputFormat.getFilter; /** This reader is used to read a {@link VectorizedColumnBatch} from input split. */ @@ -122,29 +122,34 @@ public ParquetColumnarRowSplitReader( this.utcTimestamp = utcTimestamp; this.selectedTypes = selectedTypes; this.batchSize = batchSize; - // then we need to apply the predicate push down filter - ParquetMetadata footer = - readFooter(conf, path, range(splitStart, splitStart + splitLength)); - MessageType fileSchema = footer.getFileMetaData().getSchema(); FilterCompat.Filter filter = getFilter(conf); - List blocks = filterRowGroups(filter, footer.getBlocks(), fileSchema); - - this.fileSchema = footer.getFileMetaData().getSchema(); + ParquetReadOptions parquetReadOptions = + ParquetReadOptions.builder() + .withRange(splitStart, splitStart + splitLength) + .withRecordFilter(filter) + .build(); + org.apache.flink.core.fs.Path flinkPath = + new org.apache.flink.core.fs.Path(path.toUri().toString()); + FileSystem fs = flinkPath.getFileSystem(); + ParquetInputFile inputFile = + new ParquetInputFile(fs.open(flinkPath), fs.getFileStatus(flinkPath).getLen()); + + this.reader = ParquetFileReader.open(inputFile, parquetReadOptions); + MessageType fileSchema = reader.getFooter().getFileMetaData().getSchema(); + this.fileSchema = fileSchema; this.requestedSchema = clipParquetSchema(fileSchema, selectedFieldNames, caseSensitive); - this.reader = - new ParquetFileReader( - conf, footer.getFileMetaData(), path, blocks, requestedSchema.getColumns()); + this.reader.setRequestedSchema(requestedSchema); - long totalRowCount = 0; - for (BlockMetaData block : blocks) { - totalRowCount += block.getRowCount(); - } - this.totalRowCount = totalRowCount; + this.totalRowCount = reader.getRecordCount(); this.nextRow = 0; this.rowsInBatch = 0; this.rowsReturned = 0; checkSchema(); + GeoParquetMetadataUtil.validateGeoParquetMetadata( + reader.getFooter().getFileMetaData().getKeyValueMetaData(), + requestedSchema, + selectedTypes); this.writableVectors = createWritableVectors(); this.columnarBatch = generator.generate(createReadableVectors()); diff --git a/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/vector/ParquetSplitReaderUtil.java b/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/vector/ParquetSplitReaderUtil.java index fc757738f01ddc..ed22354bb2ad70 100644 --- a/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/vector/ParquetSplitReaderUtil.java +++ b/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/vector/ParquetSplitReaderUtil.java @@ -36,6 +36,7 @@ import org.apache.flink.formats.parquet.vector.type.ParquetGroupField; import org.apache.flink.formats.parquet.vector.type.ParquetPrimitiveField; import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.TimestampData; import org.apache.flink.table.data.columnar.vector.ColumnVector; import org.apache.flink.table.data.columnar.vector.VectorizedColumnBatch; @@ -169,14 +170,17 @@ public static ColumnVector createVectorFromConstant( case VARCHAR: case BINARY: case VARBINARY: + case GEOGRAPHY: HeapBytesVector bsv = new HeapBytesVector(batchSize); if (value == null) { bsv.fillWithNulls(); } else { bsv.fill( - value instanceof byte[] - ? (byte[]) value - : value.toString().getBytes(StandardCharsets.UTF_8)); + value instanceof GeographyData + ? ((GeographyData) value).toBytes() + : value instanceof byte[] + ? (byte[]) value + : value.toString().getBytes(StandardCharsets.UTF_8)); } return bsv; case BOOLEAN: @@ -344,6 +348,7 @@ public static ColumnReader createColumnReader( case VARCHAR: case BINARY: case VARBINARY: + case GEOGRAPHY: return new BytesColumnReader( descriptors.get(0), pages.getPageReader(descriptors.get(0))); case TIMESTAMP_WITHOUT_TIME_ZONE: @@ -438,6 +443,7 @@ public static WritableColumnVector createWritableColumnVector( case VARCHAR: case BINARY: case VARBINARY: + case GEOGRAPHY: checkArgument( typeName == PrimitiveType.PrimitiveTypeName.BINARY, "Unexpected type: %s", diff --git a/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/vector/reader/NestedPrimitiveColumnReader.java b/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/vector/reader/NestedPrimitiveColumnReader.java index ceff4d118a5b55..6727a484a9b735 100644 --- a/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/vector/reader/NestedPrimitiveColumnReader.java +++ b/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/vector/reader/NestedPrimitiveColumnReader.java @@ -238,6 +238,7 @@ private Object readPrimitiveTypedRow(LogicalType category) { case VARCHAR: case BINARY: case VARBINARY: + case GEOGRAPHY: return dataColumn.readBytes(); case BOOLEAN: return dataColumn.readBoolean(); @@ -283,6 +284,7 @@ private Object dictionaryDecodeValue(LogicalType category, Integer dictionaryVal case VARCHAR: case BINARY: case VARBINARY: + case GEOGRAPHY: return dictionary.readBytes(dictionaryValue); case DATE: case TIME_WITHOUT_TIME_ZONE: @@ -324,6 +326,7 @@ private WritableColumnVector fillColumnVector(int total, List valueList) { case VARCHAR: case BINARY: case VARBINARY: + case GEOGRAPHY: HeapBytesVector heapBytesVector = new HeapBytesVector(total); for (int i = 0; i < valueList.size(); i++) { byte[] src = ((List) valueList).get(i); diff --git a/flink-formats/flink-parquet/src/test/java/org/apache/flink/formats/parquet/row/ParquetRowDataWriterTest.java b/flink-formats/flink-parquet/src/test/java/org/apache/flink/formats/parquet/row/ParquetRowDataWriterTest.java index cd0bf0016672d7..9a77f146098fcb 100644 --- a/flink-formats/flink-parquet/src/test/java/org/apache/flink/formats/parquet/row/ParquetRowDataWriterTest.java +++ b/flink-formats/flink-parquet/src/test/java/org/apache/flink/formats/parquet/row/ParquetRowDataWriterTest.java @@ -22,9 +22,14 @@ import org.apache.flink.core.fs.FileSystem; import org.apache.flink.core.fs.Path; import org.apache.flink.formats.parquet.ParquetWriterFactory; +import org.apache.flink.formats.parquet.utils.GeoParquetMetadataUtil; import org.apache.flink.formats.parquet.vector.ParquetColumnarRowSplitReader; import org.apache.flink.formats.parquet.vector.ParquetSplitReaderUtil; +import org.apache.flink.table.data.GenericArrayData; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.TimestampData; import org.apache.flink.table.data.util.DataFormatConverters; import org.apache.flink.table.types.DataType; import org.apache.flink.table.types.logical.ArrayType; @@ -33,6 +38,7 @@ import org.apache.flink.table.types.logical.DecimalType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.MapType; import org.apache.flink.table.types.logical.RowType; @@ -43,16 +49,23 @@ import org.apache.flink.table.types.logical.VarCharType; import org.apache.flink.table.types.utils.TypeConversions; import org.apache.flink.types.Row; +import org.apache.flink.util.jackson.JacksonMapperFactory; + +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; import org.apache.avro.util.Utf8; import org.apache.hadoop.conf.Configuration; import org.apache.parquet.avro.AvroParquetReader; +import org.apache.parquet.hadoop.ParquetFileReader; import org.apache.parquet.hadoop.ParquetOutputFormat; import org.apache.parquet.hadoop.ParquetReader; import org.apache.parquet.hadoop.util.HadoopInputFile; import org.apache.parquet.io.InputFile; +import org.apache.parquet.io.LocalInputFile; +import org.apache.parquet.schema.PrimitiveType; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -77,6 +90,16 @@ /** Test for {@link ParquetRowDataBuilder} and {@link ParquetRowDataWriter}. */ class ParquetRowDataWriterTest { + private static final ObjectMapper OBJECT_MAPPER = JacksonMapperFactory.createObjectMapper(); + + private static final byte[] POINT_WKB = + new byte[] { + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, (byte) 0xF0, 0x3F, 0, 0, 0, 0, 0, 0, 0, 0x40 + }; + + private static final byte[] SECOND_POINT_WKB = + new byte[] {1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x40, 0, 0, 0, 0, 0, 0, 8, 0x40}; + private static final RowType ROW_TYPE = RowType.of( new VarCharType(VarCharType.MAX_LENGTH), @@ -192,6 +215,275 @@ public void testInt64Timestamp(@TempDir java.nio.file.Path folder) throws Except invalidTypeTest(folder, conf, false); } + @Test + void testGeographyType(@TempDir java.nio.file.Path folder) throws Exception { + RowType rowType = RowType.of(new GeographyType()); + Path path = new Path(folder.toString(), UUID.randomUUID().toString()); + Configuration conf = new Configuration(); + + ParquetWriterFactory factory = + ParquetRowDataBuilder.createWriterFactory(rowType, conf, true); + BulkWriter writer = + factory.create(path.getFileSystem().create(path, FileSystem.WriteMode.OVERWRITE)); + writer.addElement(GenericRowData.of(GeographyData.fromBytes(POINT_WKB))); + writer.flush(); + writer.finish(); + + try (ParquetFileReader fileReader = + ParquetFileReader.open(new LocalInputFile(new File(path.getPath()).toPath()))) { + PrimitiveType field = + fileReader.getFileMetaData().getSchema().getType(0).asPrimitiveType(); + assertThat(field.getPrimitiveTypeName()) + .isEqualTo(PrimitiveType.PrimitiveTypeName.BINARY); + assertThat(field.getLogicalTypeAnnotation()).isNull(); + + JsonNode geoMetadata = + OBJECT_MAPPER.readTree( + fileReader + .getFileMetaData() + .getKeyValueMetaData() + .get(GeoParquetMetadataUtil.GEO_METADATA_KEY)); + assertThat(geoMetadata.get("version").asText()) + .isEqualTo(GeoParquetMetadataUtil.GEOPARQUET_VERSION); + assertThat(geoMetadata.get("primary_column").asText()).isEqualTo("f0"); + assertThat(geoMetadata.get("columns").get("f0").get("encoding").asText()) + .isEqualTo("WKB"); + assertThat(geoMetadata.get("columns").get("f0").get("geometry_types").size()).isZero(); + assertThat(geoMetadata.get("columns").get("f0").get("edges").asText()) + .isEqualTo("spherical"); + assertThat(geoMetadata.get("columns").get("f0").has("crs")).isFalse(); + } + + ParquetColumnarRowSplitReader reader = + ParquetSplitReaderUtil.genPartColumnarRowReader( + true, + true, + conf, + rowType.getFieldNames().toArray(new String[0]), + rowType.getChildren().stream() + .map(TypeConversions::fromLogicalToDataType) + .toArray(DataType[]::new), + new HashMap<>(), + IntStream.range(0, rowType.getFieldCount()).toArray(), + 50, + path, + 0, + Long.MAX_VALUE); + + assertThat(reader.reachedEnd()).isFalse(); + assertThat(reader.nextRecord().getGeography(0).toBytes()).isEqualTo(POINT_WKB); + assertThat(reader.reachedEnd()).isTrue(); + } + + @Test + void testGeoParquetMixedSchemaRoundTrip(@TempDir java.nio.file.Path folder) throws Exception { + RowType rowType = + RowType.of( + new BigIntType(), + new GeographyType(), + new VarCharType(VarCharType.MAX_LENGTH), + new BooleanType(), + new TimestampType(3), + new GeographyType()); + Path path = new Path(folder.toString(), UUID.randomUUID().toString()); + Configuration conf = new Configuration(); + + byte[] lineStringWkb = + bytesFromHex( + "01020000000300000000000000000000000000000000000000000000000000F03F" + + "000000000000F03F0000000000000040000000000000F03F"); + byte[] polygonWkb = + bytesFromHex( + "010300000001000000050000000000000000000000000000000000000000000000" + + "0000000000000000000000F03F000000000000F03F000000000000F03F" + + "000000000000F03F0000000000000000000000000000000000000000000000"); + byte[] multiPointWkb = + bytesFromHex( + "0104000000020000000101000000000000000000F03F000000000000F03F" + + "010100000000000000000000400000000000000040"); + byte[] multiLineStringWkb = + bytesFromHex( + "010500000002000000010200000002000000000000000000000000000000000000" + + "00000000000000F03F000000000000F03F010200000002000000000000" + + "0000000040000000000000004000000000000008400000000000000840"); + byte[] multiPolygonWkb = + bytesFromHex( + "010600000002000000010300000001000000050000000000000000000000000000" + + "00000000000000000000000000000000000000F03F000000000000F03F0000" + + "00000000F03F000000000000F03F0000000000000000000000000000000000" + + "00000000000000010300000001000000050000000000000000000040000000" + + "00000000400000000000000040000000000000084000000000000008400000" + + "000000000840000000000000084000000000000000400000000000000040" + + "0000000000000040"); + byte[] geometryCollectionWkb = + bytesFromHex( + "0107000000020000000101000000000000000000F03F0000000000000040" + + "010200000002000000000000000000000000000000000000000000000000" + + "00F03F000000000000F03F"); + + List rows = new ArrayList<>(); + rows.add( + GenericRowData.of( + 1L, + GeographyData.fromBytes(POINT_WKB), + org.apache.flink.table.data.StringData.fromString("Point"), + true, + TimestampData.fromLocalDateTime(LocalDateTime.of(2026, 7, 27, 10, 0)), + GeographyData.fromBytes(lineStringWkb))); + rows.add( + GenericRowData.of( + 2L, + GeographyData.fromBytes(polygonWkb), + org.apache.flink.table.data.StringData.fromString("Polygon"), + false, + TimestampData.fromLocalDateTime(LocalDateTime.of(2026, 7, 27, 10, 1)), + GeographyData.fromBytes(multiPointWkb))); + rows.add( + GenericRowData.of( + 3L, + GeographyData.fromBytes(multiLineStringWkb), + org.apache.flink.table.data.StringData.fromString("MultiLineString"), + true, + TimestampData.fromLocalDateTime(LocalDateTime.of(2026, 7, 27, 10, 2)), + GeographyData.fromBytes(multiPolygonWkb))); + rows.add( + GenericRowData.of( + 4L, + null, + org.apache.flink.table.data.StringData.fromString("GeometryCollection"), + false, + TimestampData.fromLocalDateTime(LocalDateTime.of(2026, 7, 27, 10, 3)), + GeographyData.fromBytes(geometryCollectionWkb))); + + ParquetWriterFactory factory = + ParquetRowDataBuilder.createWriterFactory(rowType, conf, true); + BulkWriter writer = + factory.create(path.getFileSystem().create(path, FileSystem.WriteMode.OVERWRITE)); + for (RowData row : rows) { + writer.addElement(row); + } + writer.flush(); + writer.finish(); + + try (ParquetFileReader fileReader = + ParquetFileReader.open(new LocalInputFile(new File(path.getPath()).toPath()))) { + JsonNode geoMetadata = + OBJECT_MAPPER.readTree( + fileReader + .getFileMetaData() + .getKeyValueMetaData() + .get(GeoParquetMetadataUtil.GEO_METADATA_KEY)); + + assertThat(geoMetadata.get("version").asText()) + .isEqualTo(GeoParquetMetadataUtil.GEOPARQUET_VERSION); + assertThat(geoMetadata.get("primary_column").asText()).isEqualTo("f1"); + assertThat(geoMetadata.get("columns").has("f1")).isTrue(); + assertThat(geoMetadata.get("columns").has("f5")).isTrue(); + assertThat(geoMetadata.get("columns").get("f1").get("encoding").asText()) + .isEqualTo("WKB"); + assertThat(geoMetadata.get("columns").get("f5").get("encoding").asText()) + .isEqualTo("WKB"); + assertThat(geoMetadata.get("columns").get("f1").get("geometry_types").size()).isZero(); + assertThat(geoMetadata.get("columns").get("f5").get("geometry_types").size()).isZero(); + assertThat(geoMetadata.get("columns").get("f1").get("edges").asText()) + .isEqualTo("spherical"); + assertThat(geoMetadata.get("columns").get("f5").get("edges").asText()) + .isEqualTo("spherical"); + } + + try (ParquetColumnarRowSplitReader reader = + ParquetSplitReaderUtil.genPartColumnarRowReader( + true, + true, + conf, + rowType.getFieldNames().toArray(new String[0]), + rowType.getChildren().stream() + .map(TypeConversions::fromLogicalToDataType) + .toArray(DataType[]::new), + new HashMap<>(), + IntStream.range(0, rowType.getFieldCount()).toArray(), + 50, + path, + 0, + Long.MAX_VALUE)) { + int index = 0; + while (!reader.reachedEnd()) { + RowData row = reader.nextRecord(); + RowData expected = rows.get(index++); + + assertThat(row.getLong(0)).isEqualTo(expected.getLong(0)); + if (expected.isNullAt(1)) { + assertThat(row.isNullAt(1)).isTrue(); + } else { + assertThat(row.getGeography(1).toBytes()) + .isEqualTo(expected.getGeography(1).toBytes()); + } + assertThat(row.getString(2).toString()).isEqualTo(expected.getString(2).toString()); + assertThat(row.getBoolean(3)).isEqualTo(expected.getBoolean(3)); + assertThat(row.getTimestamp(4, 3)).isEqualTo(expected.getTimestamp(4, 3)); + assertThat(row.getGeography(5).toBytes()) + .isEqualTo(expected.getGeography(5).toBytes()); + } + assertThat(index).isEqualTo(rows.size()); + } + } + + @Test + void testNestedGeographyType(@TempDir java.nio.file.Path folder) throws Exception { + RowType rowType = RowType.of(new BigIntType(), new ArrayType(true, new GeographyType())); + Path path = new Path(folder.toString(), UUID.randomUUID().toString()); + Configuration conf = new Configuration(); + + ParquetWriterFactory factory = + ParquetRowDataBuilder.createWriterFactory(rowType, conf, true); + BulkWriter writer = + factory.create(path.getFileSystem().create(path, FileSystem.WriteMode.OVERWRITE)); + writer.addElement( + GenericRowData.of( + 1L, + new GenericArrayData( + new Object[] { + GeographyData.fromBytes(POINT_WKB), + GeographyData.fromBytes(SECOND_POINT_WKB), + null + }))); + writer.flush(); + writer.finish(); + + try (ParquetColumnarRowSplitReader reader = + ParquetSplitReaderUtil.genPartColumnarRowReader( + true, + true, + conf, + rowType.getFieldNames().toArray(new String[0]), + rowType.getChildren().stream() + .map(TypeConversions::fromLogicalToDataType) + .toArray(DataType[]::new), + new HashMap<>(), + IntStream.range(0, rowType.getFieldCount()).toArray(), + 50, + path, + 0, + Long.MAX_VALUE)) { + assertThat(reader.reachedEnd()).isFalse(); + RowData row = reader.nextRecord(); + assertThat(row.getLong(0)).isEqualTo(1L); + assertThat(row.getArray(1).size()).isEqualTo(3); + assertThat(row.getArray(1).getGeography(0).toBytes()).isEqualTo(POINT_WKB); + assertThat(row.getArray(1).getGeography(1).toBytes()).isEqualTo(SECOND_POINT_WKB); + assertThat(row.getArray(1).isNullAt(2)).isTrue(); + assertThat(reader.reachedEnd()).isTrue(); + } + } + + private static byte[] bytesFromHex(String hex) { + byte[] bytes = new byte[hex.length() / 2]; + for (int i = 0; i < bytes.length; i++) { + bytes[i] = (byte) Integer.parseInt(hex.substring(i * 2, i * 2 + 2), 16); + } + return bytes; + } + private void innerTest(java.nio.file.Path folder, Configuration conf, boolean utcTimestamp) throws IOException { Path path = new Path(folder.toString(), UUID.randomUUID().toString()); diff --git a/flink-python/pyflink/fn_execution/coder_impl_fast.pxd b/flink-python/pyflink/fn_execution/coder_impl_fast.pxd index 05d3ba5fd6b9b3..291743ac6019a6 100644 --- a/flink-python/pyflink/fn_execution/coder_impl_fast.pxd +++ b/flink-python/pyflink/fn_execution/coder_impl_fast.pxd @@ -126,6 +126,9 @@ cdef class DoubleCoderImpl(FieldCoderImpl): cdef class BinaryCoderImpl(FieldCoderImpl): pass +cdef class GeographyCoderImpl(FieldCoderImpl): + pass + cdef class CharCoderImpl(FieldCoderImpl): pass diff --git a/flink-python/pyflink/fn_execution/coder_impl_fast.pyx b/flink-python/pyflink/fn_execution/coder_impl_fast.pyx index 92dff893fe922b..37d5daac8d57e1 100644 --- a/flink-python/pyflink/fn_execution/coder_impl_fast.pyx +++ b/flink-python/pyflink/fn_execution/coder_impl_fast.pyx @@ -585,6 +585,25 @@ cdef class BinaryCoderImpl(FieldCoderImpl): cpdef decode_from_stream(self, InputStream in_stream, size_t size): return in_stream.read_bytes() + +cdef class GeographyCoderImpl(FieldCoderImpl): + """ + A coder compatible with GeographyTypeSerializer's versioned WKB format. + """ + + cpdef encode_to_stream(self, value, OutputStream out_stream): + out_stream.write_byte(1) + out_stream.write_bytes(value, len(value)) + + cpdef decode_from_stream(self, InputStream in_stream, size_t size): + cdef long format_version = in_stream.read_byte() + if format_version != 1: + raise ValueError( + "Unsupported GEOGRAPHY serializer format version %d. Expected 1." + % format_version) + return in_stream.read_bytes() + + cdef class CharCoderImpl(FieldCoderImpl): """ A coder for a str value. diff --git a/flink-python/pyflink/fn_execution/coder_impl_slow.py b/flink-python/pyflink/fn_execution/coder_impl_slow.py index 769720dc277194..854435ef117867 100644 --- a/flink-python/pyflink/fn_execution/coder_impl_slow.py +++ b/flink-python/pyflink/fn_execution/coder_impl_slow.py @@ -441,6 +441,26 @@ def decode_from_stream(self, in_stream: InputStream, length=0): return in_stream.read_bytes() +class GeographyCoderImpl(FieldCoderImpl): + """ + A coder compatible with GeographyTypeSerializer's versioned WKB format. + """ + + FORMAT_VERSION = 1 + + def encode_to_stream(self, value, out_stream: OutputStream): + out_stream.write_byte(self.FORMAT_VERSION) + out_stream.write_bytes(value, len(value)) + + def decode_from_stream(self, in_stream: InputStream, length=0): + format_version = in_stream.read_byte() + if format_version != self.FORMAT_VERSION: + raise ValueError( + "Unsupported GEOGRAPHY serializer format version %d. Expected %d." + % (format_version, self.FORMAT_VERSION)) + return in_stream.read_bytes() + + class CharCoderImpl(FieldCoderImpl): """ A coder for a str value. diff --git a/flink-python/pyflink/fn_execution/coders.py b/flink-python/pyflink/fn_execution/coders.py index 838528f24d1202..ba29e91a9a181d 100644 --- a/flink-python/pyflink/fn_execution/coders.py +++ b/flink-python/pyflink/fn_execution/coders.py @@ -37,12 +37,13 @@ from pyflink.table.types import TinyIntType, SmallIntType, IntType, BigIntType, BooleanType, \ FloatType, DoubleType, VarCharType, VarBinaryType, DecimalType, DateType, TimeType, \ LocalZonedTimestampType, RowType, RowField, to_arrow_type, TimestampType, ArrayType, MapType, \ - BinaryType, NullType, CharType + BinaryType, GeographyType, NullType, CharType __all__ = ['FlattenRowCoder', 'RowCoder', 'BigIntCoder', 'TinyIntCoder', 'BooleanCoder', - 'SmallIntCoder', 'IntCoder', 'FloatCoder', 'DoubleCoder', 'BinaryCoder', 'CharCoder', - 'DateCoder', 'TimeCoder', 'TimestampCoder', 'LocalZonedTimestampCoder', 'InstantCoder', - 'GenericArrayCoder', 'PrimitiveArrayCoder', 'MapCoder', 'DecimalCoder', + 'SmallIntCoder', 'IntCoder', 'FloatCoder', 'DoubleCoder', 'BinaryCoder', + 'GeographyCoder', 'CharCoder', 'DateCoder', 'TimeCoder', 'TimestampCoder', + 'LocalZonedTimestampCoder', 'InstantCoder', 'GenericArrayCoder', + 'PrimitiveArrayCoder', 'MapCoder', 'DecimalCoder', 'BigDecimalCoder', 'TupleCoder', 'TimeWindowCoder', 'CountWindowCoder', 'PickleCoder', 'CloudPickleCoder', 'DataViewFilterCoder'] @@ -134,6 +135,8 @@ def _to_data_type(cls, field_type): return BinaryType(field_type.binary_info.length, field_type.nullable) elif field_type.type_name == flink_fn_execution_pb2.Schema.VARBINARY: return VarBinaryType(field_type.var_binary_info.length, field_type.nullable) + elif field_type.type_name == flink_fn_execution_pb2.Schema.GEOGRAPHY: + return GeographyType(field_type.nullable) elif field_type.type_name == flink_fn_execution_pb2.Schema.DECIMAL: return DecimalType(field_type.decimal_info.precision, field_type.decimal_info.scale, @@ -476,6 +479,15 @@ def get_impl(self): return coder_impl.BinaryCoderImpl() +class GeographyCoder(FieldCoder): + """ + Coder for GEOGRAPHY values represented as WKB bytes. + """ + + def get_impl(self): + return coder_impl.GeographyCoderImpl() + + class CharCoder(FieldCoder): """ Coder for Character String. @@ -673,6 +685,7 @@ def from_proto(field_type): type_name.DOUBLE: DoubleCoder(), type_name.BINARY: BinaryCoder(), type_name.VARBINARY: BinaryCoder(), + type_name.GEOGRAPHY: GeographyCoder(), type_name.CHAR: CharCoder(), type_name.VARCHAR: CharCoder(), type_name.DATE: DateCoder(), diff --git a/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.py b/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.py index 96984f80d5c777..444dfb44274161 100644 --- a/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.py +++ b/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.py @@ -31,7 +31,7 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18\x66link-fn-execution.proto\x12 org.apache.flink.fn_execution.v1\"*\n\x0cJobParameter\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"\x86\x01\n\x05Input\x12\x44\n\x03udf\x18\x01 \x01(\x0b\x32\x35.org.apache.flink.fn_execution.v1.UserDefinedFunctionH\x00\x12\x15\n\x0binputOffset\x18\x02 \x01(\x05H\x00\x12\x17\n\rinputConstant\x18\x03 \x01(\x0cH\x00\x42\x07\n\x05input\"\xa8\x01\n\x13UserDefinedFunction\x12\x0f\n\x07payload\x18\x01 \x01(\x0c\x12\x37\n\x06inputs\x18\x02 \x03(\x0b\x32\'.org.apache.flink.fn_execution.v1.Input\x12\x14\n\x0cwindow_index\x18\x03 \x01(\x05\x12\x1a\n\x12takes_row_as_input\x18\x04 \x01(\x08\x12\x15\n\ris_pandas_udf\x18\x05 \x01(\x08\"\x90\x01\n\x0c\x41syncOptions\x12!\n\x19max_concurrent_operations\x18\x01 \x01(\x05\x12\x12\n\ntimeout_ms\x18\x02 \x01(\x03\x12\x15\n\rretry_enabled\x18\x03 \x01(\x08\x12\x1a\n\x12retry_max_attempts\x18\x04 \x01(\x05\x12\x16\n\x0eretry_delay_ms\x18\x05 \x01(\x03\"\xc3\x03\n\x14UserDefinedFunctions\x12\x43\n\x04udfs\x18\x01 \x03(\x0b\x32\x35.org.apache.flink.fn_execution.v1.UserDefinedFunction\x12\x16\n\x0emetric_enabled\x18\x02 \x01(\x08\x12=\n\x07windows\x18\x03 \x03(\x0b\x32,.org.apache.flink.fn_execution.v1.OverWindow\x12\x17\n\x0fprofile_enabled\x18\x04 \x01(\x08\x12\x46\n\x0ejob_parameters\x18\x05 \x03(\x0b\x32..org.apache.flink.fn_execution.v1.JobParameter\x12\x45\n\rasync_options\x18\x06 \x01(\x0b\x32..org.apache.flink.fn_execution.v1.AsyncOptions\x12g\n\x0fruntime_context\x18\x07 \x01(\x0b\x32N.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.RuntimeContext\"\xdd\x02\n\nOverWindow\x12L\n\x0bwindow_type\x18\x01 \x01(\x0e\x32\x37.org.apache.flink.fn_execution.v1.OverWindow.WindowType\x12\x16\n\x0elower_boundary\x18\x02 \x01(\x03\x12\x16\n\x0eupper_boundary\x18\x03 \x01(\x03\"\xd0\x01\n\nWindowType\x12\x13\n\x0fRANGE_UNBOUNDED\x10\x00\x12\x1d\n\x19RANGE_UNBOUNDED_PRECEDING\x10\x01\x12\x1d\n\x19RANGE_UNBOUNDED_FOLLOWING\x10\x02\x12\x11\n\rRANGE_SLIDING\x10\x03\x12\x11\n\rROW_UNBOUNDED\x10\x04\x12\x1b\n\x17ROW_UNBOUNDED_PRECEDING\x10\x05\x12\x1b\n\x17ROW_UNBOUNDED_FOLLOWING\x10\x06\x12\x0f\n\x0bROW_SLIDING\x10\x07\"\x8b\x06\n\x1cUserDefinedAggregateFunction\x12\x0f\n\x07payload\x18\x01 \x01(\x0c\x12\x37\n\x06inputs\x18\x02 \x03(\x0b\x32\'.org.apache.flink.fn_execution.v1.Input\x12Z\n\x05specs\x18\x03 \x03(\x0b\x32K.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction.DataViewSpec\x12\x12\n\nfilter_arg\x18\x04 \x01(\x05\x12\x10\n\x08\x64istinct\x18\x05 \x01(\x08\x12\x1a\n\x12takes_row_as_input\x18\x06 \x01(\x08\x1a\x82\x04\n\x0c\x44\x61taViewSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x66ield_index\x18\x02 \x01(\x05\x12i\n\tlist_view\x18\x03 \x01(\x0b\x32T.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction.DataViewSpec.ListViewH\x00\x12g\n\x08map_view\x18\x04 \x01(\x0b\x32S.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction.DataViewSpec.MapViewH\x00\x1aT\n\x08ListView\x12H\n\x0c\x65lement_type\x18\x01 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x1a\x97\x01\n\x07MapView\x12\x44\n\x08key_type\x18\x01 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x46\n\nvalue_type\x18\x02 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldTypeB\x0b\n\tdata_view\"\xac\x04\n\x0bGroupWindow\x12M\n\x0bwindow_type\x18\x01 \x01(\x0e\x32\x38.org.apache.flink.fn_execution.v1.GroupWindow.WindowType\x12\x16\n\x0eis_time_window\x18\x02 \x01(\x08\x12\x14\n\x0cwindow_slide\x18\x03 \x01(\x03\x12\x13\n\x0bwindow_size\x18\x04 \x01(\x03\x12\x12\n\nwindow_gap\x18\x05 \x01(\x03\x12\x13\n\x0bis_row_time\x18\x06 \x01(\x08\x12\x18\n\x10time_field_index\x18\x07 \x01(\x05\x12\x17\n\x0f\x61llowedLateness\x18\x08 \x01(\x03\x12U\n\x0fnamedProperties\x18\t \x03(\x0e\x32<.org.apache.flink.fn_execution.v1.GroupWindow.WindowProperty\x12\x16\n\x0eshift_timezone\x18\n \x01(\t\"[\n\nWindowType\x12\x19\n\x15TUMBLING_GROUP_WINDOW\x10\x00\x12\x18\n\x14SLIDING_GROUP_WINDOW\x10\x01\x12\x18\n\x14SESSION_GROUP_WINDOW\x10\x02\"c\n\x0eWindowProperty\x12\x10\n\x0cWINDOW_START\x10\x00\x12\x0e\n\nWINDOW_END\x10\x01\x12\x16\n\x12ROW_TIME_ATTRIBUTE\x10\x02\x12\x17\n\x13PROC_TIME_ATTRIBUTE\x10\x03\"\xc7\x05\n\x1dUserDefinedAggregateFunctions\x12L\n\x04udfs\x18\x01 \x03(\x0b\x32>.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction\x12\x16\n\x0emetric_enabled\x18\x02 \x01(\x08\x12\x10\n\x08grouping\x18\x03 \x03(\x05\x12\x1e\n\x16generate_update_before\x18\x04 \x01(\x08\x12\x44\n\x08key_type\x18\x05 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x1b\n\x13index_of_count_star\x18\x06 \x01(\x05\x12\x1e\n\x16state_cleaning_enabled\x18\x07 \x01(\x08\x12\x18\n\x10state_cache_size\x18\x08 \x01(\x05\x12!\n\x19map_state_read_cache_size\x18\t \x01(\x05\x12\"\n\x1amap_state_write_cache_size\x18\n \x01(\x05\x12\x1b\n\x13\x63ount_star_inserted\x18\x0b \x01(\x08\x12\x43\n\x0cgroup_window\x18\x0c \x01(\x0b\x32-.org.apache.flink.fn_execution.v1.GroupWindow\x12\x17\n\x0fprofile_enabled\x18\r \x01(\x08\x12\x46\n\x0ejob_parameters\x18\x0e \x03(\x0b\x32..org.apache.flink.fn_execution.v1.JobParameter\x12g\n\x0fruntime_context\x18\x0f \x01(\x0b\x32N.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.RuntimeContext\"\xf6\x0f\n\x06Schema\x12>\n\x06\x66ields\x18\x01 \x03(\x0b\x32..org.apache.flink.fn_execution.v1.Schema.Field\x1a\x97\x01\n\x07MapInfo\x12\x44\n\x08key_type\x18\x01 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x46\n\nvalue_type\x18\x02 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x1a\x1d\n\x08TimeInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a\"\n\rTimestampInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a,\n\x17LocalZonedTimestampInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a\'\n\x12ZonedTimestampInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a/\n\x0b\x44\x65\x63imalInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x12\r\n\x05scale\x18\x02 \x01(\x05\x1a\x1c\n\nBinaryInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\x1f\n\rVarBinaryInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\x1a\n\x08\x43harInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\x1d\n\x0bVarCharInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\xb0\x08\n\tFieldType\x12\x44\n\ttype_name\x18\x01 \x01(\x0e\x32\x31.org.apache.flink.fn_execution.v1.Schema.TypeName\x12\x10\n\x08nullable\x18\x02 \x01(\x08\x12U\n\x17\x63ollection_element_type\x18\x03 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldTypeH\x00\x12\x44\n\x08map_info\x18\x04 \x01(\x0b\x32\x30.org.apache.flink.fn_execution.v1.Schema.MapInfoH\x00\x12>\n\nrow_schema\x18\x05 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.SchemaH\x00\x12L\n\x0c\x64\x65\x63imal_info\x18\x06 \x01(\x0b\x32\x34.org.apache.flink.fn_execution.v1.Schema.DecimalInfoH\x00\x12\x46\n\ttime_info\x18\x07 \x01(\x0b\x32\x31.org.apache.flink.fn_execution.v1.Schema.TimeInfoH\x00\x12P\n\x0etimestamp_info\x18\x08 \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.Schema.TimestampInfoH\x00\x12\x66\n\x1alocal_zoned_timestamp_info\x18\t \x01(\x0b\x32@.org.apache.flink.fn_execution.v1.Schema.LocalZonedTimestampInfoH\x00\x12[\n\x14zoned_timestamp_info\x18\n \x01(\x0b\x32;.org.apache.flink.fn_execution.v1.Schema.ZonedTimestampInfoH\x00\x12J\n\x0b\x62inary_info\x18\x0b \x01(\x0b\x32\x33.org.apache.flink.fn_execution.v1.Schema.BinaryInfoH\x00\x12Q\n\x0fvar_binary_info\x18\x0c \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.Schema.VarBinaryInfoH\x00\x12\x46\n\tchar_info\x18\r \x01(\x0b\x32\x31.org.apache.flink.fn_execution.v1.Schema.CharInfoH\x00\x12M\n\rvar_char_info\x18\x0e \x01(\x0b\x32\x34.org.apache.flink.fn_execution.v1.Schema.VarCharInfoH\x00\x42\x0b\n\ttype_info\x1al\n\x05\x46ield\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12@\n\x04type\x18\x03 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\"\xab\x02\n\x08TypeName\x12\x07\n\x03ROW\x10\x00\x12\x0b\n\x07TINYINT\x10\x01\x12\x0c\n\x08SMALLINT\x10\x02\x12\x07\n\x03INT\x10\x03\x12\n\n\x06\x42IGINT\x10\x04\x12\x0b\n\x07\x44\x45\x43IMAL\x10\x05\x12\t\n\x05\x46LOAT\x10\x06\x12\n\n\x06\x44OUBLE\x10\x07\x12\x08\n\x04\x44\x41TE\x10\x08\x12\x08\n\x04TIME\x10\t\x12\r\n\tTIMESTAMP\x10\n\x12\x0b\n\x07\x42OOLEAN\x10\x0b\x12\n\n\x06\x42INARY\x10\x0c\x12\r\n\tVARBINARY\x10\r\x12\x08\n\x04\x43HAR\x10\x0e\x12\x0b\n\x07VARCHAR\x10\x0f\x12\x0f\n\x0b\x42\x41SIC_ARRAY\x10\x10\x12\x07\n\x03MAP\x10\x11\x12\x0c\n\x08MULTISET\x10\x12\x12\x19\n\x15LOCAL_ZONED_TIMESTAMP\x10\x13\x12\x13\n\x0fZONED_TIMESTAMP\x10\x14\x12\x08\n\x04NULL\x10\x15\"\xc3\n\n\x08TypeInfo\x12\x46\n\ttype_name\x18\x01 \x01(\x0e\x32\x33.org.apache.flink.fn_execution.v1.TypeInfo.TypeName\x12M\n\x17\x63ollection_element_type\x18\x02 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfoH\x00\x12O\n\rrow_type_info\x18\x03 \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.TypeInfo.RowTypeInfoH\x00\x12S\n\x0ftuple_type_info\x18\x04 \x01(\x0b\x32\x38.org.apache.flink.fn_execution.v1.TypeInfo.TupleTypeInfoH\x00\x12O\n\rmap_type_info\x18\x05 \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.TypeInfo.MapTypeInfoH\x00\x12Q\n\x0e\x61vro_type_info\x18\x06 \x01(\x0b\x32\x37.org.apache.flink.fn_execution.v1.TypeInfo.AvroTypeInfoH\x00\x1a\x8b\x01\n\x0bMapTypeInfo\x12<\n\x08key_type\x18\x01 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x12>\n\nvalue_type\x18\x02 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x1a\xb8\x01\n\x0bRowTypeInfo\x12L\n\x06\x66ields\x18\x01 \x03(\x0b\x32<.org.apache.flink.fn_execution.v1.TypeInfo.RowTypeInfo.Field\x1a[\n\x05\x46ield\x12\x12\n\nfield_name\x18\x01 \x01(\t\x12>\n\nfield_type\x18\x02 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x1aP\n\rTupleTypeInfo\x12?\n\x0b\x66ield_types\x18\x01 \x03(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x1a\x1e\n\x0c\x41vroTypeInfo\x12\x0e\n\x06schema\x18\x01 \x01(\t\"\x8d\x03\n\x08TypeName\x12\x07\n\x03ROW\x10\x00\x12\n\n\x06STRING\x10\x01\x12\x08\n\x04\x42YTE\x10\x02\x12\x0b\n\x07\x42OOLEAN\x10\x03\x12\t\n\x05SHORT\x10\x04\x12\x07\n\x03INT\x10\x05\x12\x08\n\x04LONG\x10\x06\x12\t\n\x05\x46LOAT\x10\x07\x12\n\n\x06\x44OUBLE\x10\x08\x12\x08\n\x04\x43HAR\x10\t\x12\x0b\n\x07\x42IG_INT\x10\n\x12\x0b\n\x07\x42IG_DEC\x10\x0b\x12\x0c\n\x08SQL_DATE\x10\x0c\x12\x0c\n\x08SQL_TIME\x10\r\x12\x11\n\rSQL_TIMESTAMP\x10\x0e\x12\x0f\n\x0b\x42\x41SIC_ARRAY\x10\x0f\x12\x13\n\x0fPRIMITIVE_ARRAY\x10\x10\x12\t\n\x05TUPLE\x10\x11\x12\x08\n\x04LIST\x10\x12\x12\x07\n\x03MAP\x10\x13\x12\x11\n\rPICKLED_BYTES\x10\x14\x12\x10\n\x0cOBJECT_ARRAY\x10\x15\x12\x0b\n\x07INSTANT\x10\x16\x12\x08\n\x04\x41VRO\x10\x17\x12\x0e\n\nLOCAL_DATE\x10\x18\x12\x0e\n\nLOCAL_TIME\x10\x19\x12\x12\n\x0eLOCAL_DATETIME\x10\x1a\x12\x19\n\x15LOCAL_ZONED_TIMESTAMP\x10\x1b\x42\x0b\n\ttype_info\"\xd1\x07\n\x1dUserDefinedDataStreamFunction\x12\x63\n\rfunction_type\x18\x01 \x01(\x0e\x32L.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.FunctionType\x12g\n\x0fruntime_context\x18\x02 \x01(\x0b\x32N.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.RuntimeContext\x12\x0f\n\x07payload\x18\x03 \x01(\x0c\x12\x16\n\x0emetric_enabled\x18\x04 \x01(\x08\x12\x41\n\rkey_type_info\x18\x05 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x12\x17\n\x0fprofile_enabled\x18\x06 \x01(\x08\x12\x17\n\x0fhas_side_output\x18\x07 \x01(\x08\x12\x18\n\x10state_cache_size\x18\x08 \x01(\x05\x12!\n\x19map_state_read_cache_size\x18\t \x01(\x05\x12\"\n\x1amap_state_write_cache_size\x18\n \x01(\x05\x1a\xb2\x02\n\x0eRuntimeContext\x12\x11\n\ttask_name\x18\x01 \x01(\t\x12\x1f\n\x17task_name_with_subtasks\x18\x02 \x01(\t\x12#\n\x1bnumber_of_parallel_subtasks\x18\x03 \x01(\x05\x12\'\n\x1fmax_number_of_parallel_subtasks\x18\x04 \x01(\x05\x12\x1d\n\x15index_of_this_subtask\x18\x05 \x01(\x05\x12\x16\n\x0e\x61ttempt_number\x18\x06 \x01(\x05\x12\x46\n\x0ejob_parameters\x18\x07 \x03(\x0b\x32..org.apache.flink.fn_execution.v1.JobParameter\x12\x1f\n\x17in_batch_execution_mode\x18\x08 \x01(\x08\"\xad\x01\n\x0c\x46unctionType\x12\x0b\n\x07PROCESS\x10\x00\x12\x0e\n\nCO_PROCESS\x10\x01\x12\x11\n\rKEYED_PROCESS\x10\x02\x12\x14\n\x10KEYED_CO_PROCESS\x10\x03\x12\n\n\x06WINDOW\x10\x04\x12\x18\n\x14\x43O_BROADCAST_PROCESS\x10\x05\x12\x1e\n\x1aKEYED_CO_BROADCAST_PROCESS\x10\x06\x12\x11\n\rREVISE_OUTPUT\x10\x64\"\xe4\x0e\n\x0fStateDescriptor\x12\x12\n\nstate_name\x18\x01 \x01(\t\x12Z\n\x10state_ttl_config\x18\x02 \x01(\x0b\x32@.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig\x1a\xe0\r\n\x0eStateTTLConfig\x12`\n\x0bupdate_type\x18\x01 \x01(\x0e\x32K.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.UpdateType\x12j\n\x10state_visibility\x18\x02 \x01(\x0e\x32P.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.StateVisibility\x12w\n\x17ttl_time_characteristic\x18\x03 \x01(\x0e\x32V.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.TtlTimeCharacteristic\x12\x0b\n\x03ttl\x18\x04 \x01(\x03\x12n\n\x12\x63leanup_strategies\x18\x05 \x01(\x0b\x32R.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies\x1a\xca\x08\n\x11\x43leanupStrategies\x12 \n\x18is_cleanup_in_background\x18\x01 \x01(\x08\x12y\n\nstrategies\x18\x02 \x03(\x0b\x32\x65.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.MapStrategiesEntry\x1aX\n\x1aIncrementalCleanupStrategy\x12\x14\n\x0c\x63leanup_size\x18\x01 \x01(\x05\x12$\n\x1crun_cleanup_for_every_record\x18\x02 \x01(\x08\x1aK\n#RocksdbCompactFilterCleanupStrategy\x12$\n\x1cquery_time_after_num_entries\x18\x01 \x01(\x03\x1a\xe0\x04\n\x12MapStrategiesEntry\x12o\n\x08strategy\x18\x01 \x01(\x0e\x32].org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.Strategies\x12\x81\x01\n\x0e\x65mpty_strategy\x18\x02 \x01(\x0e\x32g.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.EmptyCleanupStrategyH\x00\x12\x95\x01\n\x1cincremental_cleanup_strategy\x18\x03 \x01(\x0b\x32m.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.IncrementalCleanupStrategyH\x00\x12\xa9\x01\n\'rocksdb_compact_filter_cleanup_strategy\x18\x04 \x01(\x0b\x32v.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.RocksdbCompactFilterCleanupStrategyH\x00\x42\x11\n\x0f\x43leanupStrategy\"b\n\nStrategies\x12\x1c\n\x18\x46ULL_STATE_SCAN_SNAPSHOT\x10\x00\x12\x17\n\x13INCREMENTAL_CLEANUP\x10\x01\x12\x1d\n\x19ROCKSDB_COMPACTION_FILTER\x10\x02\"*\n\x14\x45mptyCleanupStrategy\x12\x12\n\x0e\x45MPTY_STRATEGY\x10\x00\"D\n\nUpdateType\x12\x0c\n\x08\x44isabled\x10\x00\x12\x14\n\x10OnCreateAndWrite\x10\x01\x12\x12\n\x0eOnReadAndWrite\x10\x02\"J\n\x0fStateVisibility\x12\x1f\n\x1bReturnExpiredIfNotCleanedUp\x10\x00\x12\x16\n\x12NeverReturnExpired\x10\x01\"+\n\x15TtlTimeCharacteristic\x12\x12\n\x0eProcessingTime\x10\x00\"\xf1\x07\n\x13\x43oderInfoDescriptor\x12`\n\x10\x66latten_row_type\x18\x01 \x01(\x0b\x32\x44.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.FlattenRowTypeH\x00\x12Q\n\x08row_type\x18\x02 \x01(\x0b\x32=.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.RowTypeH\x00\x12U\n\narrow_type\x18\x03 \x01(\x0b\x32?.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.ArrowTypeH\x00\x12k\n\x16over_window_arrow_type\x18\x04 \x01(\x0b\x32I.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.OverWindowArrowTypeH\x00\x12Q\n\x08raw_type\x18\x05 \x01(\x0b\x32=.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.RawTypeH\x00\x12H\n\x04mode\x18\x06 \x01(\x0e\x32:.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.Mode\x12\"\n\x1aseparated_with_end_message\x18\x07 \x01(\x08\x1aJ\n\x0e\x46lattenRowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1a\x43\n\x07RowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1a\x45\n\tArrowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1aO\n\x13OverWindowArrowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1aH\n\x07RawType\x12=\n\ttype_info\x18\x01 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\" \n\x04Mode\x12\n\n\x06SINGLE\x10\x00\x12\x0c\n\x08MULTIPLE\x10\x01\x42\x0b\n\tdata_typeB-\n\x1forg.apache.flink.fnexecution.v1B\nFlinkFnApib\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18\x66link-fn-execution.proto\x12 org.apache.flink.fn_execution.v1\"*\n\x0cJobParameter\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"\x86\x01\n\x05Input\x12\x44\n\x03udf\x18\x01 \x01(\x0b\x32\x35.org.apache.flink.fn_execution.v1.UserDefinedFunctionH\x00\x12\x15\n\x0binputOffset\x18\x02 \x01(\x05H\x00\x12\x17\n\rinputConstant\x18\x03 \x01(\x0cH\x00\x42\x07\n\x05input\"\xa8\x01\n\x13UserDefinedFunction\x12\x0f\n\x07payload\x18\x01 \x01(\x0c\x12\x37\n\x06inputs\x18\x02 \x03(\x0b\x32\'.org.apache.flink.fn_execution.v1.Input\x12\x14\n\x0cwindow_index\x18\x03 \x01(\x05\x12\x1a\n\x12takes_row_as_input\x18\x04 \x01(\x08\x12\x15\n\ris_pandas_udf\x18\x05 \x01(\x08\"\x90\x01\n\x0c\x41syncOptions\x12!\n\x19max_concurrent_operations\x18\x01 \x01(\x05\x12\x12\n\ntimeout_ms\x18\x02 \x01(\x03\x12\x15\n\rretry_enabled\x18\x03 \x01(\x08\x12\x1a\n\x12retry_max_attempts\x18\x04 \x01(\x05\x12\x16\n\x0eretry_delay_ms\x18\x05 \x01(\x03\"\xc3\x03\n\x14UserDefinedFunctions\x12\x43\n\x04udfs\x18\x01 \x03(\x0b\x32\x35.org.apache.flink.fn_execution.v1.UserDefinedFunction\x12\x16\n\x0emetric_enabled\x18\x02 \x01(\x08\x12=\n\x07windows\x18\x03 \x03(\x0b\x32,.org.apache.flink.fn_execution.v1.OverWindow\x12\x17\n\x0fprofile_enabled\x18\x04 \x01(\x08\x12\x46\n\x0ejob_parameters\x18\x05 \x03(\x0b\x32..org.apache.flink.fn_execution.v1.JobParameter\x12\x45\n\rasync_options\x18\x06 \x01(\x0b\x32..org.apache.flink.fn_execution.v1.AsyncOptions\x12g\n\x0fruntime_context\x18\x07 \x01(\x0b\x32N.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.RuntimeContext\"\xdd\x02\n\nOverWindow\x12L\n\x0bwindow_type\x18\x01 \x01(\x0e\x32\x37.org.apache.flink.fn_execution.v1.OverWindow.WindowType\x12\x16\n\x0elower_boundary\x18\x02 \x01(\x03\x12\x16\n\x0eupper_boundary\x18\x03 \x01(\x03\"\xd0\x01\n\nWindowType\x12\x13\n\x0fRANGE_UNBOUNDED\x10\x00\x12\x1d\n\x19RANGE_UNBOUNDED_PRECEDING\x10\x01\x12\x1d\n\x19RANGE_UNBOUNDED_FOLLOWING\x10\x02\x12\x11\n\rRANGE_SLIDING\x10\x03\x12\x11\n\rROW_UNBOUNDED\x10\x04\x12\x1b\n\x17ROW_UNBOUNDED_PRECEDING\x10\x05\x12\x1b\n\x17ROW_UNBOUNDED_FOLLOWING\x10\x06\x12\x0f\n\x0bROW_SLIDING\x10\x07\"\x8b\x06\n\x1cUserDefinedAggregateFunction\x12\x0f\n\x07payload\x18\x01 \x01(\x0c\x12\x37\n\x06inputs\x18\x02 \x03(\x0b\x32\'.org.apache.flink.fn_execution.v1.Input\x12Z\n\x05specs\x18\x03 \x03(\x0b\x32K.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction.DataViewSpec\x12\x12\n\nfilter_arg\x18\x04 \x01(\x05\x12\x10\n\x08\x64istinct\x18\x05 \x01(\x08\x12\x1a\n\x12takes_row_as_input\x18\x06 \x01(\x08\x1a\x82\x04\n\x0c\x44\x61taViewSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x66ield_index\x18\x02 \x01(\x05\x12i\n\tlist_view\x18\x03 \x01(\x0b\x32T.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction.DataViewSpec.ListViewH\x00\x12g\n\x08map_view\x18\x04 \x01(\x0b\x32S.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction.DataViewSpec.MapViewH\x00\x1aT\n\x08ListView\x12H\n\x0c\x65lement_type\x18\x01 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x1a\x97\x01\n\x07MapView\x12\x44\n\x08key_type\x18\x01 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x46\n\nvalue_type\x18\x02 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldTypeB\x0b\n\tdata_view\"\xac\x04\n\x0bGroupWindow\x12M\n\x0bwindow_type\x18\x01 \x01(\x0e\x32\x38.org.apache.flink.fn_execution.v1.GroupWindow.WindowType\x12\x16\n\x0eis_time_window\x18\x02 \x01(\x08\x12\x14\n\x0cwindow_slide\x18\x03 \x01(\x03\x12\x13\n\x0bwindow_size\x18\x04 \x01(\x03\x12\x12\n\nwindow_gap\x18\x05 \x01(\x03\x12\x13\n\x0bis_row_time\x18\x06 \x01(\x08\x12\x18\n\x10time_field_index\x18\x07 \x01(\x05\x12\x17\n\x0f\x61llowedLateness\x18\x08 \x01(\x03\x12U\n\x0fnamedProperties\x18\t \x03(\x0e\x32<.org.apache.flink.fn_execution.v1.GroupWindow.WindowProperty\x12\x16\n\x0eshift_timezone\x18\n \x01(\t\"[\n\nWindowType\x12\x19\n\x15TUMBLING_GROUP_WINDOW\x10\x00\x12\x18\n\x14SLIDING_GROUP_WINDOW\x10\x01\x12\x18\n\x14SESSION_GROUP_WINDOW\x10\x02\"c\n\x0eWindowProperty\x12\x10\n\x0cWINDOW_START\x10\x00\x12\x0e\n\nWINDOW_END\x10\x01\x12\x16\n\x12ROW_TIME_ATTRIBUTE\x10\x02\x12\x17\n\x13PROC_TIME_ATTRIBUTE\x10\x03\"\xc7\x05\n\x1dUserDefinedAggregateFunctions\x12L\n\x04udfs\x18\x01 \x03(\x0b\x32>.org.apache.flink.fn_execution.v1.UserDefinedAggregateFunction\x12\x16\n\x0emetric_enabled\x18\x02 \x01(\x08\x12\x10\n\x08grouping\x18\x03 \x03(\x05\x12\x1e\n\x16generate_update_before\x18\x04 \x01(\x08\x12\x44\n\x08key_type\x18\x05 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x1b\n\x13index_of_count_star\x18\x06 \x01(\x05\x12\x1e\n\x16state_cleaning_enabled\x18\x07 \x01(\x08\x12\x18\n\x10state_cache_size\x18\x08 \x01(\x05\x12!\n\x19map_state_read_cache_size\x18\t \x01(\x05\x12\"\n\x1amap_state_write_cache_size\x18\n \x01(\x05\x12\x1b\n\x13\x63ount_star_inserted\x18\x0b \x01(\x08\x12\x43\n\x0cgroup_window\x18\x0c \x01(\x0b\x32-.org.apache.flink.fn_execution.v1.GroupWindow\x12\x17\n\x0fprofile_enabled\x18\r \x01(\x08\x12\x46\n\x0ejob_parameters\x18\x0e \x03(\x0b\x32..org.apache.flink.fn_execution.v1.JobParameter\x12g\n\x0fruntime_context\x18\x0f \x01(\x0b\x32N.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.RuntimeContext\"\x85\x10\n\x06Schema\x12>\n\x06\x66ields\x18\x01 \x03(\x0b\x32..org.apache.flink.fn_execution.v1.Schema.Field\x1a\x97\x01\n\x07MapInfo\x12\x44\n\x08key_type\x18\x01 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x46\n\nvalue_type\x18\x02 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x1a\x1d\n\x08TimeInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a\"\n\rTimestampInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a,\n\x17LocalZonedTimestampInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a\'\n\x12ZonedTimestampInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x1a/\n\x0b\x44\x65\x63imalInfo\x12\x11\n\tprecision\x18\x01 \x01(\x05\x12\r\n\x05scale\x18\x02 \x01(\x05\x1a\x1c\n\nBinaryInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\x1f\n\rVarBinaryInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\x1a\n\x08\x43harInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\x1d\n\x0bVarCharInfo\x12\x0e\n\x06length\x18\x01 \x01(\x05\x1a\xb0\x08\n\tFieldType\x12\x44\n\ttype_name\x18\x01 \x01(\x0e\x32\x31.org.apache.flink.fn_execution.v1.Schema.TypeName\x12\x10\n\x08nullable\x18\x02 \x01(\x08\x12U\n\x17\x63ollection_element_type\x18\x03 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldTypeH\x00\x12\x44\n\x08map_info\x18\x04 \x01(\x0b\x32\x30.org.apache.flink.fn_execution.v1.Schema.MapInfoH\x00\x12>\n\nrow_schema\x18\x05 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.SchemaH\x00\x12L\n\x0c\x64\x65\x63imal_info\x18\x06 \x01(\x0b\x32\x34.org.apache.flink.fn_execution.v1.Schema.DecimalInfoH\x00\x12\x46\n\ttime_info\x18\x07 \x01(\x0b\x32\x31.org.apache.flink.fn_execution.v1.Schema.TimeInfoH\x00\x12P\n\x0etimestamp_info\x18\x08 \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.Schema.TimestampInfoH\x00\x12\x66\n\x1alocal_zoned_timestamp_info\x18\t \x01(\x0b\x32@.org.apache.flink.fn_execution.v1.Schema.LocalZonedTimestampInfoH\x00\x12[\n\x14zoned_timestamp_info\x18\n \x01(\x0b\x32;.org.apache.flink.fn_execution.v1.Schema.ZonedTimestampInfoH\x00\x12J\n\x0b\x62inary_info\x18\x0b \x01(\x0b\x32\x33.org.apache.flink.fn_execution.v1.Schema.BinaryInfoH\x00\x12Q\n\x0fvar_binary_info\x18\x0c \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.Schema.VarBinaryInfoH\x00\x12\x46\n\tchar_info\x18\r \x01(\x0b\x32\x31.org.apache.flink.fn_execution.v1.Schema.CharInfoH\x00\x12M\n\rvar_char_info\x18\x0e \x01(\x0b\x32\x34.org.apache.flink.fn_execution.v1.Schema.VarCharInfoH\x00\x42\x0b\n\ttype_info\x1al\n\x05\x46ield\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12@\n\x04type\x18\x03 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\"\xba\x02\n\x08TypeName\x12\x07\n\x03ROW\x10\x00\x12\x0b\n\x07TINYINT\x10\x01\x12\x0c\n\x08SMALLINT\x10\x02\x12\x07\n\x03INT\x10\x03\x12\n\n\x06\x42IGINT\x10\x04\x12\x0b\n\x07\x44\x45\x43IMAL\x10\x05\x12\t\n\x05\x46LOAT\x10\x06\x12\n\n\x06\x44OUBLE\x10\x07\x12\x08\n\x04\x44\x41TE\x10\x08\x12\x08\n\x04TIME\x10\t\x12\r\n\tTIMESTAMP\x10\n\x12\x0b\n\x07\x42OOLEAN\x10\x0b\x12\n\n\x06\x42INARY\x10\x0c\x12\r\n\tVARBINARY\x10\r\x12\x08\n\x04\x43HAR\x10\x0e\x12\x0b\n\x07VARCHAR\x10\x0f\x12\x0f\n\x0b\x42\x41SIC_ARRAY\x10\x10\x12\x07\n\x03MAP\x10\x11\x12\x0c\n\x08MULTISET\x10\x12\x12\x19\n\x15LOCAL_ZONED_TIMESTAMP\x10\x13\x12\x13\n\x0fZONED_TIMESTAMP\x10\x14\x12\x08\n\x04NULL\x10\x15\x12\r\n\tGEOGRAPHY\x10\x16\"\xc3\n\n\x08TypeInfo\x12\x46\n\ttype_name\x18\x01 \x01(\x0e\x32\x33.org.apache.flink.fn_execution.v1.TypeInfo.TypeName\x12M\n\x17\x63ollection_element_type\x18\x02 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfoH\x00\x12O\n\rrow_type_info\x18\x03 \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.TypeInfo.RowTypeInfoH\x00\x12S\n\x0ftuple_type_info\x18\x04 \x01(\x0b\x32\x38.org.apache.flink.fn_execution.v1.TypeInfo.TupleTypeInfoH\x00\x12O\n\rmap_type_info\x18\x05 \x01(\x0b\x32\x36.org.apache.flink.fn_execution.v1.TypeInfo.MapTypeInfoH\x00\x12Q\n\x0e\x61vro_type_info\x18\x06 \x01(\x0b\x32\x37.org.apache.flink.fn_execution.v1.TypeInfo.AvroTypeInfoH\x00\x1a\x8b\x01\n\x0bMapTypeInfo\x12<\n\x08key_type\x18\x01 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x12>\n\nvalue_type\x18\x02 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x1a\xb8\x01\n\x0bRowTypeInfo\x12L\n\x06\x66ields\x18\x01 \x03(\x0b\x32<.org.apache.flink.fn_execution.v1.TypeInfo.RowTypeInfo.Field\x1a[\n\x05\x46ield\x12\x12\n\nfield_name\x18\x01 \x01(\t\x12>\n\nfield_type\x18\x02 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x1aP\n\rTupleTypeInfo\x12?\n\x0b\x66ield_types\x18\x01 \x03(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x1a\x1e\n\x0c\x41vroTypeInfo\x12\x0e\n\x06schema\x18\x01 \x01(\t\"\x8d\x03\n\x08TypeName\x12\x07\n\x03ROW\x10\x00\x12\n\n\x06STRING\x10\x01\x12\x08\n\x04\x42YTE\x10\x02\x12\x0b\n\x07\x42OOLEAN\x10\x03\x12\t\n\x05SHORT\x10\x04\x12\x07\n\x03INT\x10\x05\x12\x08\n\x04LONG\x10\x06\x12\t\n\x05\x46LOAT\x10\x07\x12\n\n\x06\x44OUBLE\x10\x08\x12\x08\n\x04\x43HAR\x10\t\x12\x0b\n\x07\x42IG_INT\x10\n\x12\x0b\n\x07\x42IG_DEC\x10\x0b\x12\x0c\n\x08SQL_DATE\x10\x0c\x12\x0c\n\x08SQL_TIME\x10\r\x12\x11\n\rSQL_TIMESTAMP\x10\x0e\x12\x0f\n\x0b\x42\x41SIC_ARRAY\x10\x0f\x12\x13\n\x0fPRIMITIVE_ARRAY\x10\x10\x12\t\n\x05TUPLE\x10\x11\x12\x08\n\x04LIST\x10\x12\x12\x07\n\x03MAP\x10\x13\x12\x11\n\rPICKLED_BYTES\x10\x14\x12\x10\n\x0cOBJECT_ARRAY\x10\x15\x12\x0b\n\x07INSTANT\x10\x16\x12\x08\n\x04\x41VRO\x10\x17\x12\x0e\n\nLOCAL_DATE\x10\x18\x12\x0e\n\nLOCAL_TIME\x10\x19\x12\x12\n\x0eLOCAL_DATETIME\x10\x1a\x12\x19\n\x15LOCAL_ZONED_TIMESTAMP\x10\x1b\x42\x0b\n\ttype_info\"\xd1\x07\n\x1dUserDefinedDataStreamFunction\x12\x63\n\rfunction_type\x18\x01 \x01(\x0e\x32L.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.FunctionType\x12g\n\x0fruntime_context\x18\x02 \x01(\x0b\x32N.org.apache.flink.fn_execution.v1.UserDefinedDataStreamFunction.RuntimeContext\x12\x0f\n\x07payload\x18\x03 \x01(\x0c\x12\x16\n\x0emetric_enabled\x18\x04 \x01(\x08\x12\x41\n\rkey_type_info\x18\x05 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\x12\x17\n\x0fprofile_enabled\x18\x06 \x01(\x08\x12\x17\n\x0fhas_side_output\x18\x07 \x01(\x08\x12\x18\n\x10state_cache_size\x18\x08 \x01(\x05\x12!\n\x19map_state_read_cache_size\x18\t \x01(\x05\x12\"\n\x1amap_state_write_cache_size\x18\n \x01(\x05\x1a\xb2\x02\n\x0eRuntimeContext\x12\x11\n\ttask_name\x18\x01 \x01(\t\x12\x1f\n\x17task_name_with_subtasks\x18\x02 \x01(\t\x12#\n\x1bnumber_of_parallel_subtasks\x18\x03 \x01(\x05\x12\'\n\x1fmax_number_of_parallel_subtasks\x18\x04 \x01(\x05\x12\x1d\n\x15index_of_this_subtask\x18\x05 \x01(\x05\x12\x16\n\x0e\x61ttempt_number\x18\x06 \x01(\x05\x12\x46\n\x0ejob_parameters\x18\x07 \x03(\x0b\x32..org.apache.flink.fn_execution.v1.JobParameter\x12\x1f\n\x17in_batch_execution_mode\x18\x08 \x01(\x08\"\xad\x01\n\x0c\x46unctionType\x12\x0b\n\x07PROCESS\x10\x00\x12\x0e\n\nCO_PROCESS\x10\x01\x12\x11\n\rKEYED_PROCESS\x10\x02\x12\x14\n\x10KEYED_CO_PROCESS\x10\x03\x12\n\n\x06WINDOW\x10\x04\x12\x18\n\x14\x43O_BROADCAST_PROCESS\x10\x05\x12\x1e\n\x1aKEYED_CO_BROADCAST_PROCESS\x10\x06\x12\x11\n\rREVISE_OUTPUT\x10\x64\"\xe4\x0e\n\x0fStateDescriptor\x12\x12\n\nstate_name\x18\x01 \x01(\t\x12Z\n\x10state_ttl_config\x18\x02 \x01(\x0b\x32@.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig\x1a\xe0\r\n\x0eStateTTLConfig\x12`\n\x0bupdate_type\x18\x01 \x01(\x0e\x32K.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.UpdateType\x12j\n\x10state_visibility\x18\x02 \x01(\x0e\x32P.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.StateVisibility\x12w\n\x17ttl_time_characteristic\x18\x03 \x01(\x0e\x32V.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.TtlTimeCharacteristic\x12\x0b\n\x03ttl\x18\x04 \x01(\x03\x12n\n\x12\x63leanup_strategies\x18\x05 \x01(\x0b\x32R.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies\x1a\xca\x08\n\x11\x43leanupStrategies\x12 \n\x18is_cleanup_in_background\x18\x01 \x01(\x08\x12y\n\nstrategies\x18\x02 \x03(\x0b\x32\x65.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.MapStrategiesEntry\x1aX\n\x1aIncrementalCleanupStrategy\x12\x14\n\x0c\x63leanup_size\x18\x01 \x01(\x05\x12$\n\x1crun_cleanup_for_every_record\x18\x02 \x01(\x08\x1aK\n#RocksdbCompactFilterCleanupStrategy\x12$\n\x1cquery_time_after_num_entries\x18\x01 \x01(\x03\x1a\xe0\x04\n\x12MapStrategiesEntry\x12o\n\x08strategy\x18\x01 \x01(\x0e\x32].org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.Strategies\x12\x81\x01\n\x0e\x65mpty_strategy\x18\x02 \x01(\x0e\x32g.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.EmptyCleanupStrategyH\x00\x12\x95\x01\n\x1cincremental_cleanup_strategy\x18\x03 \x01(\x0b\x32m.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.IncrementalCleanupStrategyH\x00\x12\xa9\x01\n\'rocksdb_compact_filter_cleanup_strategy\x18\x04 \x01(\x0b\x32v.org.apache.flink.fn_execution.v1.StateDescriptor.StateTTLConfig.CleanupStrategies.RocksdbCompactFilterCleanupStrategyH\x00\x42\x11\n\x0f\x43leanupStrategy\"b\n\nStrategies\x12\x1c\n\x18\x46ULL_STATE_SCAN_SNAPSHOT\x10\x00\x12\x17\n\x13INCREMENTAL_CLEANUP\x10\x01\x12\x1d\n\x19ROCKSDB_COMPACTION_FILTER\x10\x02\"*\n\x14\x45mptyCleanupStrategy\x12\x12\n\x0e\x45MPTY_STRATEGY\x10\x00\"D\n\nUpdateType\x12\x0c\n\x08\x44isabled\x10\x00\x12\x14\n\x10OnCreateAndWrite\x10\x01\x12\x12\n\x0eOnReadAndWrite\x10\x02\"J\n\x0fStateVisibility\x12\x1f\n\x1bReturnExpiredIfNotCleanedUp\x10\x00\x12\x16\n\x12NeverReturnExpired\x10\x01\"+\n\x15TtlTimeCharacteristic\x12\x12\n\x0eProcessingTime\x10\x00\"\xf1\x07\n\x13\x43oderInfoDescriptor\x12`\n\x10\x66latten_row_type\x18\x01 \x01(\x0b\x32\x44.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.FlattenRowTypeH\x00\x12Q\n\x08row_type\x18\x02 \x01(\x0b\x32=.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.RowTypeH\x00\x12U\n\narrow_type\x18\x03 \x01(\x0b\x32?.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.ArrowTypeH\x00\x12k\n\x16over_window_arrow_type\x18\x04 \x01(\x0b\x32I.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.OverWindowArrowTypeH\x00\x12Q\n\x08raw_type\x18\x05 \x01(\x0b\x32=.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.RawTypeH\x00\x12H\n\x04mode\x18\x06 \x01(\x0e\x32:.org.apache.flink.fn_execution.v1.CoderInfoDescriptor.Mode\x12\"\n\x1aseparated_with_end_message\x18\x07 \x01(\x08\x1aJ\n\x0e\x46lattenRowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1a\x43\n\x07RowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1a\x45\n\tArrowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1aO\n\x13OverWindowArrowType\x12\x38\n\x06schema\x18\x01 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.Schema\x1aH\n\x07RawType\x12=\n\ttype_info\x18\x01 \x01(\x0b\x32*.org.apache.flink.fn_execution.v1.TypeInfo\" \n\x04Mode\x12\n\n\x06SINGLE\x10\x00\x12\x0c\n\x08MULTIPLE\x10\x01\x42\x0b\n\tdata_typeB-\n\x1forg.apache.flink.fnexecution.v1B\nFlinkFnApib\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -70,7 +70,7 @@ _globals['_USERDEFINEDAGGREGATEFUNCTIONS']._serialized_start=2709 _globals['_USERDEFINEDAGGREGATEFUNCTIONS']._serialized_end=3420 _globals['_SCHEMA']._serialized_start=3423 - _globals['_SCHEMA']._serialized_end=5461 + _globals['_SCHEMA']._serialized_end=5476 _globals['_SCHEMA_MAPINFO']._serialized_start=3498 _globals['_SCHEMA_MAPINFO']._serialized_end=3649 _globals['_SCHEMA_TIMEINFO']._serialized_start=3651 @@ -96,61 +96,61 @@ _globals['_SCHEMA_FIELD']._serialized_start=5051 _globals['_SCHEMA_FIELD']._serialized_end=5159 _globals['_SCHEMA_TYPENAME']._serialized_start=5162 - _globals['_SCHEMA_TYPENAME']._serialized_end=5461 - _globals['_TYPEINFO']._serialized_start=5464 - _globals['_TYPEINFO']._serialized_end=6811 - _globals['_TYPEINFO_MAPTYPEINFO']._serialized_start=5958 - _globals['_TYPEINFO_MAPTYPEINFO']._serialized_end=6097 - _globals['_TYPEINFO_ROWTYPEINFO']._serialized_start=6100 - _globals['_TYPEINFO_ROWTYPEINFO']._serialized_end=6284 - _globals['_TYPEINFO_ROWTYPEINFO_FIELD']._serialized_start=6193 - _globals['_TYPEINFO_ROWTYPEINFO_FIELD']._serialized_end=6284 - _globals['_TYPEINFO_TUPLETYPEINFO']._serialized_start=6286 - _globals['_TYPEINFO_TUPLETYPEINFO']._serialized_end=6366 - _globals['_TYPEINFO_AVROTYPEINFO']._serialized_start=6368 - _globals['_TYPEINFO_AVROTYPEINFO']._serialized_end=6398 - _globals['_TYPEINFO_TYPENAME']._serialized_start=6401 - _globals['_TYPEINFO_TYPENAME']._serialized_end=6798 - _globals['_USERDEFINEDDATASTREAMFUNCTION']._serialized_start=6814 - _globals['_USERDEFINEDDATASTREAMFUNCTION']._serialized_end=7791 - _globals['_USERDEFINEDDATASTREAMFUNCTION_RUNTIMECONTEXT']._serialized_start=7309 - _globals['_USERDEFINEDDATASTREAMFUNCTION_RUNTIMECONTEXT']._serialized_end=7615 - _globals['_USERDEFINEDDATASTREAMFUNCTION_FUNCTIONTYPE']._serialized_start=7618 - _globals['_USERDEFINEDDATASTREAMFUNCTION_FUNCTIONTYPE']._serialized_end=7791 - _globals['_STATEDESCRIPTOR']._serialized_start=7794 - _globals['_STATEDESCRIPTOR']._serialized_end=9686 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG']._serialized_start=7926 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG']._serialized_end=9686 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES']._serialized_start=8397 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES']._serialized_end=9495 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_INCREMENTALCLEANUPSTRATEGY']._serialized_start=8575 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_INCREMENTALCLEANUPSTRATEGY']._serialized_end=8663 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_ROCKSDBCOMPACTFILTERCLEANUPSTRATEGY']._serialized_start=8665 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_ROCKSDBCOMPACTFILTERCLEANUPSTRATEGY']._serialized_end=8740 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_MAPSTRATEGIESENTRY']._serialized_start=8743 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_MAPSTRATEGIESENTRY']._serialized_end=9351 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_STRATEGIES']._serialized_start=9353 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_STRATEGIES']._serialized_end=9451 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_EMPTYCLEANUPSTRATEGY']._serialized_start=9453 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_EMPTYCLEANUPSTRATEGY']._serialized_end=9495 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_UPDATETYPE']._serialized_start=9497 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_UPDATETYPE']._serialized_end=9565 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_STATEVISIBILITY']._serialized_start=9567 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_STATEVISIBILITY']._serialized_end=9641 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_TTLTIMECHARACTERISTIC']._serialized_start=9643 - _globals['_STATEDESCRIPTOR_STATETTLCONFIG_TTLTIMECHARACTERISTIC']._serialized_end=9686 - _globals['_CODERINFODESCRIPTOR']._serialized_start=9689 - _globals['_CODERINFODESCRIPTOR']._serialized_end=10698 - _globals['_CODERINFODESCRIPTOR_FLATTENROWTYPE']._serialized_start=10282 - _globals['_CODERINFODESCRIPTOR_FLATTENROWTYPE']._serialized_end=10356 - _globals['_CODERINFODESCRIPTOR_ROWTYPE']._serialized_start=10358 - _globals['_CODERINFODESCRIPTOR_ROWTYPE']._serialized_end=10425 - _globals['_CODERINFODESCRIPTOR_ARROWTYPE']._serialized_start=10427 - _globals['_CODERINFODESCRIPTOR_ARROWTYPE']._serialized_end=10496 - _globals['_CODERINFODESCRIPTOR_OVERWINDOWARROWTYPE']._serialized_start=10498 - _globals['_CODERINFODESCRIPTOR_OVERWINDOWARROWTYPE']._serialized_end=10577 - _globals['_CODERINFODESCRIPTOR_RAWTYPE']._serialized_start=10579 - _globals['_CODERINFODESCRIPTOR_RAWTYPE']._serialized_end=10651 - _globals['_CODERINFODESCRIPTOR_MODE']._serialized_start=10653 - _globals['_CODERINFODESCRIPTOR_MODE']._serialized_end=10685 + _globals['_SCHEMA_TYPENAME']._serialized_end=5476 + _globals['_TYPEINFO']._serialized_start=5479 + _globals['_TYPEINFO']._serialized_end=6826 + _globals['_TYPEINFO_MAPTYPEINFO']._serialized_start=5973 + _globals['_TYPEINFO_MAPTYPEINFO']._serialized_end=6112 + _globals['_TYPEINFO_ROWTYPEINFO']._serialized_start=6115 + _globals['_TYPEINFO_ROWTYPEINFO']._serialized_end=6299 + _globals['_TYPEINFO_ROWTYPEINFO_FIELD']._serialized_start=6208 + _globals['_TYPEINFO_ROWTYPEINFO_FIELD']._serialized_end=6299 + _globals['_TYPEINFO_TUPLETYPEINFO']._serialized_start=6301 + _globals['_TYPEINFO_TUPLETYPEINFO']._serialized_end=6381 + _globals['_TYPEINFO_AVROTYPEINFO']._serialized_start=6383 + _globals['_TYPEINFO_AVROTYPEINFO']._serialized_end=6413 + _globals['_TYPEINFO_TYPENAME']._serialized_start=6416 + _globals['_TYPEINFO_TYPENAME']._serialized_end=6813 + _globals['_USERDEFINEDDATASTREAMFUNCTION']._serialized_start=6829 + _globals['_USERDEFINEDDATASTREAMFUNCTION']._serialized_end=7806 + _globals['_USERDEFINEDDATASTREAMFUNCTION_RUNTIMECONTEXT']._serialized_start=7324 + _globals['_USERDEFINEDDATASTREAMFUNCTION_RUNTIMECONTEXT']._serialized_end=7630 + _globals['_USERDEFINEDDATASTREAMFUNCTION_FUNCTIONTYPE']._serialized_start=7633 + _globals['_USERDEFINEDDATASTREAMFUNCTION_FUNCTIONTYPE']._serialized_end=7806 + _globals['_STATEDESCRIPTOR']._serialized_start=7809 + _globals['_STATEDESCRIPTOR']._serialized_end=9701 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG']._serialized_start=7941 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG']._serialized_end=9701 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES']._serialized_start=8412 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES']._serialized_end=9510 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_INCREMENTALCLEANUPSTRATEGY']._serialized_start=8590 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_INCREMENTALCLEANUPSTRATEGY']._serialized_end=8678 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_ROCKSDBCOMPACTFILTERCLEANUPSTRATEGY']._serialized_start=8680 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_ROCKSDBCOMPACTFILTERCLEANUPSTRATEGY']._serialized_end=8755 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_MAPSTRATEGIESENTRY']._serialized_start=8758 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_MAPSTRATEGIESENTRY']._serialized_end=9366 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_STRATEGIES']._serialized_start=9368 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_STRATEGIES']._serialized_end=9466 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_EMPTYCLEANUPSTRATEGY']._serialized_start=9468 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_CLEANUPSTRATEGIES_EMPTYCLEANUPSTRATEGY']._serialized_end=9510 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_UPDATETYPE']._serialized_start=9512 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_UPDATETYPE']._serialized_end=9580 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_STATEVISIBILITY']._serialized_start=9582 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_STATEVISIBILITY']._serialized_end=9656 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_TTLTIMECHARACTERISTIC']._serialized_start=9658 + _globals['_STATEDESCRIPTOR_STATETTLCONFIG_TTLTIMECHARACTERISTIC']._serialized_end=9701 + _globals['_CODERINFODESCRIPTOR']._serialized_start=9704 + _globals['_CODERINFODESCRIPTOR']._serialized_end=10713 + _globals['_CODERINFODESCRIPTOR_FLATTENROWTYPE']._serialized_start=10297 + _globals['_CODERINFODESCRIPTOR_FLATTENROWTYPE']._serialized_end=10371 + _globals['_CODERINFODESCRIPTOR_ROWTYPE']._serialized_start=10373 + _globals['_CODERINFODESCRIPTOR_ROWTYPE']._serialized_end=10440 + _globals['_CODERINFODESCRIPTOR_ARROWTYPE']._serialized_start=10442 + _globals['_CODERINFODESCRIPTOR_ARROWTYPE']._serialized_end=10511 + _globals['_CODERINFODESCRIPTOR_OVERWINDOWARROWTYPE']._serialized_start=10513 + _globals['_CODERINFODESCRIPTOR_OVERWINDOWARROWTYPE']._serialized_end=10592 + _globals['_CODERINFODESCRIPTOR_RAWTYPE']._serialized_start=10594 + _globals['_CODERINFODESCRIPTOR_RAWTYPE']._serialized_end=10666 + _globals['_CODERINFODESCRIPTOR_MODE']._serialized_start=10668 + _globals['_CODERINFODESCRIPTOR_MODE']._serialized_end=10700 # @@protoc_insertion_point(module_scope) diff --git a/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.pyi b/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.pyi index b6b77c6894609e..0af6f9f8e05206 100644 --- a/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.pyi +++ b/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.pyi @@ -256,6 +256,7 @@ class Schema(_message.Message): LOCAL_ZONED_TIMESTAMP: _ClassVar[Schema.TypeName] ZONED_TIMESTAMP: _ClassVar[Schema.TypeName] NULL: _ClassVar[Schema.TypeName] + GEOGRAPHY: _ClassVar[Schema.TypeName] ROW: Schema.TypeName TINYINT: Schema.TypeName SMALLINT: Schema.TypeName @@ -278,6 +279,7 @@ class Schema(_message.Message): LOCAL_ZONED_TIMESTAMP: Schema.TypeName ZONED_TIMESTAMP: Schema.TypeName NULL: Schema.TypeName + GEOGRAPHY: Schema.TypeName class MapInfo(_message.Message): __slots__ = ("key_type", "value_type") KEY_TYPE_FIELD_NUMBER: _ClassVar[int] diff --git a/flink-python/pyflink/fn_execution/tests/test_coders.py b/flink-python/pyflink/fn_execution/tests/test_coders.py index f04a33f164999b..24b3b5fdf1d738 100644 --- a/flink-python/pyflink/fn_execution/tests/test_coders.py +++ b/flink-python/pyflink/fn_execution/tests/test_coders.py @@ -25,8 +25,9 @@ SmallIntCoder, IntCoder, FloatCoder, DoubleCoder, BinaryCoder, CharCoder, DateCoder, \ TimeCoder, TimestampCoder, GenericArrayCoder, MapCoder, DecimalCoder, FlattenRowCoder, \ RowCoder, LocalZonedTimestampCoder, BigDecimalCoder, TupleCoder, PrimitiveArrayCoder, \ - TimeWindowCoder, CountWindowCoder, InstantCoder + TimeWindowCoder, CountWindowCoder, InstantCoder, GeographyCoder, from_proto from pyflink.datastream.window import TimeWindow, CountWindow +from pyflink.fn_execution import flink_fn_execution_pb2 from pyflink.testing.test_case_utils import PyFlinkTestCase @@ -78,6 +79,15 @@ def test_binary_coder(self): coder = BinaryCoder() self.check_coder(coder, b'pyflink') + def test_geography_coder_from_proto(self): + field_type = flink_fn_execution_pb2.Schema.FieldType( + type_name=flink_fn_execution_pb2.Schema.GEOGRAPHY) + coder = from_proto(field_type) + self.assertIsInstance(coder, GeographyCoder) + self.assertEqual( + b'\x01\x00\x00\x00\tgeography', coder.get_impl().encode(b'geography')) + self.check_coder(coder, b'geography') + def test_char_coder(self): coder = CharCoder() self.check_coder(coder, 'flink', '🐿') diff --git a/flink-python/pyflink/proto/flink-fn-execution.proto b/flink-python/pyflink/proto/flink-fn-execution.proto index 4ab5616b011b88..2f832d9144cdad 100644 --- a/flink-python/pyflink/proto/flink-fn-execution.proto +++ b/flink-python/pyflink/proto/flink-fn-execution.proto @@ -233,6 +233,7 @@ message Schema { LOCAL_ZONED_TIMESTAMP = 19; ZONED_TIMESTAMP = 20; NULL = 21; + GEOGRAPHY = 22; } message MapInfo { diff --git a/flink-python/pyflink/table/tests/test_pandas_udf.py b/flink-python/pyflink/table/tests/test_pandas_udf.py index 7a6b36f5b17f7c..3210ad72171bbd 100644 --- a/flink-python/pyflink/table/tests/test_pandas_udf.py +++ b/flink-python/pyflink/table/tests/test_pandas_udf.py @@ -381,6 +381,24 @@ def local_zoned_timestamp_func(local_zoned_timestamp_param): actual = source_sink_utils.results() self.assert_equals(actual, ["+I[1970-01-02T00:00:00.123Z]"]) + def test_geography_type(self): + point_wkb = bytes([ + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xF0, 0x3F, 0, 0, 0, 0, 0, 0, 0, 0x40 + ]) + + geography_identity = udf( + lambda geography: geography, + result_type=DataTypes.GEOGRAPHY(), + func_type="pandas") + + table = self.t_env.from_elements( + [(point_wkb,)], + DataTypes.ROW([DataTypes.FIELD("g", DataTypes.GEOGRAPHY())])) + collected = list(table.select(geography_identity(table.g)).execute().collect()) + + self.assertEqual(1, len(collected)) + self.assertEqual(point_wkb, collected[0][0]) + class BatchPandasUDFITTests(PandasUDFITTests, PyFlinkBatchTableTestCase): diff --git a/flink-python/pyflink/table/tests/test_schema.py b/flink-python/pyflink/table/tests/test_schema.py index 46968c9fad3249..345eef8a7748ef 100644 --- a/flink-python/pyflink/table/tests/test_schema.py +++ b/flink-python/pyflink/table/tests/test_schema.py @@ -23,33 +23,40 @@ class SchemaTest(PyFlinkTestCase): def test_schema_basic(self): - old_schema = Schema.new_builder() \ - .from_row_data_type(DataTypes.ROW( - [DataTypes.FIELD("a", DataTypes.TINYINT()), - DataTypes.FIELD("b", DataTypes.SMALLINT()), - DataTypes.FIELD("c", DataTypes.INT())])) \ - .from_fields(["d", "e"], [DataTypes.STRING(), DataTypes.BOOLEAN()]) \ - .build() - self.schema = Schema.new_builder() \ - .from_schema(old_schema) \ - .primary_key_named("primary_constraint", "id") \ - .column("id", DataTypes.INT().not_null()) \ - .column("counter", DataTypes.INT().not_null()) \ - .column("payload", "ROW") \ - .column_by_metadata("topic", DataTypes.STRING(), None, True) \ - .column_by_expression("ts", call_sql("orig_ts - INTERVAL '60' MINUTE")) \ - .column_by_metadata("orig_ts", DataTypes.TIMESTAMP(3), "timestamp") \ - .watermark("ts", "ts - INTERVAL '5' SECOND") \ - .column_by_expression("proctime", "PROCTIME()") \ - .build() + old_schema = ( + Schema.new_builder() + .from_row_data_type( + DataTypes.ROW( + [DataTypes.FIELD("a", DataTypes.TINYINT()), + DataTypes.FIELD("b", DataTypes.SMALLINT()), + DataTypes.FIELD("c", DataTypes.INT()), + DataTypes.FIELD("f", DataTypes.GEOGRAPHY())])) + .from_fields(["d", "e"], [DataTypes.STRING(), DataTypes.BOOLEAN()]) + .build()) + self.schema = ( + Schema.new_builder() + .from_schema(old_schema) + .primary_key_named("primary_constraint", "id") + .column("id", DataTypes.INT().not_null()) + .column("counter", DataTypes.INT().not_null()) + .column("location", DataTypes.GEOGRAPHY()) + .column("payload", "ROW") + .column_by_metadata("topic", DataTypes.STRING(), None, True) + .column_by_expression("ts", call_sql("orig_ts - INTERVAL '60' MINUTE")) + .column_by_metadata("orig_ts", DataTypes.TIMESTAMP(3), "timestamp") + .watermark("ts", "ts - INTERVAL '5' SECOND") + .column_by_expression("proctime", "PROCTIME()") + .build()) self.assertEqual("""( `a` TINYINT, `b` SMALLINT, `c` INT, + `f` GEOGRAPHY, `d` STRING, `e` BOOLEAN, `id` INT NOT NULL, `counter` INT NOT NULL, + `location` GEOGRAPHY, `payload` [ROW], `topic` STRING METADATA VIRTUAL, `ts` AS [orig_ts - INTERVAL '60' MINUTE], diff --git a/flink-python/pyflink/table/tests/test_sql.py b/flink-python/pyflink/table/tests/test_sql.py index 549c48133d60a9..0455772c17222b 100644 --- a/flink-python/pyflink/table/tests/test_sql.py +++ b/flink-python/pyflink/table/tests/test_sql.py @@ -21,8 +21,9 @@ from pyflink.find_flink_home import _find_flink_source_root from pyflink.java_gateway import get_gateway -from pyflink.table import ResultKind, ExplainDetail +from pyflink.table import DataTypes, ResultKind, ExplainDetail from pyflink.table import expressions as expr +from pyflink.table.udf import udf from pyflink.testing import source_sink_utils from pyflink.testing.test_case_utils import PyFlinkStreamTableTestCase, \ PyFlinkTestCase @@ -100,6 +101,55 @@ def test_execute_sql(self): self.assertEqual(table_result.get_result_kind(), ResultKind.SUCCESS) table_result.print() + def test_geography_sql_wkb_bytes_round_trip(self): + point_wkb = bytes([ + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xF0, 0x3F, 0, 0, 0, 0, 0, 0, 0, 0x40 + ]) + source = self.t_env.from_elements( + [(point_wkb, 'POINT (1 2)')], + ['wkb', 'wkt']) + self.t_env.create_temporary_view('geography_source', source) + + result = self.t_env.sql_query( + 'SELECT ST_ASTEXT(ST_GEOGFROMWKB(wkb)), ' + 'ST_ASWKB(ST_GEOGFROMTEXT(wkt)), ' + 'ST_ASWKB(ST_GEOGFROMWKB(wkb)) ' + 'FROM geography_source') + collected = list(result.execute().collect()) + + self.assertEqual(1, len(collected)) + self.assertEqual('POINT (1 2)', collected[0][0]) + self.assertEqual(point_wkb, collected[0][1]) + self.assertEqual(point_wkb, collected[0][2]) + + def test_geography_from_elements_collect_round_trip(self): + point_wkb = bytes([ + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xF0, 0x3F, 0, 0, 0, 0, 0, 0, 0, 0x40 + ]) + + table = self.t_env.from_elements( + [(point_wkb,)], + DataTypes.ROW([DataTypes.FIELD("g", DataTypes.GEOGRAPHY())])) + collected = list(table.execute().collect()) + + self.assertEqual(1, len(collected)) + self.assertEqual(point_wkb, collected[0][0]) + + def test_geography_python_udf_round_trip(self): + point_wkb = bytes([ + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xF0, 0x3F, 0, 0, 0, 0, 0, 0, 0, 0x40 + ]) + + geography_identity = udf(lambda geography: geography, result_type=DataTypes.GEOGRAPHY()) + + table = self.t_env.from_elements( + [(point_wkb,)], + DataTypes.ROW([DataTypes.FIELD("g", DataTypes.GEOGRAPHY())])) + collected = list(table.select(geography_identity(table.g)).execute().collect()) + + self.assertEqual(1, len(collected)) + self.assertEqual(point_wkb, collected[0][0]) + class JavaSqlTests(PyFlinkTestCase): """ diff --git a/flink-python/pyflink/table/tests/test_table_environment_api.py b/flink-python/pyflink/table/tests/test_table_environment_api.py index 505d1dbd26fb4b..b11ac88dbafe20 100644 --- a/flink-python/pyflink/table/tests/test_table_environment_api.py +++ b/flink-python/pyflink/table/tests/test_table_environment_api.py @@ -697,7 +697,7 @@ def test_collect_with_retract(self): def test_collect_for_all_data_types(self): expected_result = [Row(1, None, 1, True, 32767, -2147483648, 1.23, - 1.98932, bytearray(b'pyflink'), 'pyflink', + 1.98932, b'pyflink', 'pyflink', datetime.date(2014, 9, 13), datetime.time(12, 0, 0, 123000), datetime.datetime(2018, 3, 11, 3, 0, 0, 123000), [['a', 'b'], ['c', 'd'], ['e', 'f']], diff --git a/flink-python/pyflink/table/tests/test_types.py b/flink-python/pyflink/table/tests/test_types.py index d3bd37c6ffd8fa..42004a178b7fc4 100644 --- a/flink-python/pyflink/table/tests/test_types.py +++ b/flink-python/pyflink/table/tests/test_types.py @@ -677,7 +677,14 @@ def __init__(self, **kwargs): (decimal.Decimal("1.0"), DataTypes.DECIMAL(10, 0)), # Binary + (b"\x01", DataTypes.BINARY(1)), (bytearray([1]), DataTypes.BINARY(1)), + (b"\x01\x02", DataTypes.VARBINARY(2)), + + # Geography + (bytes([ + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xF0, 0x3F, + 0, 0, 0, 0, 0, 0, 0, 0x40]), DataTypes.GEOGRAPHY()), # Date/Time/Timestamp (datetime.date(2000, 1, 2), DataTypes.DATE()), @@ -806,6 +813,7 @@ def test_basic_type(self): test_types = [DataTypes.STRING(), DataTypes.BOOLEAN(), DataTypes.BYTES(), + DataTypes.GEOGRAPHY(), DataTypes.TINYINT(), DataTypes.SMALLINT(), DataTypes.INT(), @@ -831,7 +839,8 @@ def test_atomic_type_with_data_type_with_parameters(self): JDataTypes.BINARY(2).notNull(), JDataTypes.VARCHAR(30).notNull(), JDataTypes.CHAR(50).notNull(), - JDataTypes.DECIMAL(20, 10).notNull()] + JDataTypes.DECIMAL(20, 10).notNull(), + JDataTypes.GEOGRAPHY().notNull()] converted_python_types = [_from_java_data_type(item) for item in java_types] @@ -841,7 +850,8 @@ def test_atomic_type_with_data_type_with_parameters(self): DataTypes.BINARY(2, False), DataTypes.VARCHAR(30, False), DataTypes.CHAR(50, False), - DataTypes.DECIMAL(20, 10, False)] + DataTypes.DECIMAL(20, 10, False), + DataTypes.GEOGRAPHY(False)] self.assertEqual(converted_python_types, expected) def test_array_type(self): diff --git a/flink-python/pyflink/table/types.py b/flink-python/pyflink/table/types.py index b62b55af9e1f3c..d97d8db720d48d 100644 --- a/flink-python/pyflink/table/types.py +++ b/flink-python/pyflink/table/types.py @@ -228,6 +228,15 @@ def __init__(self, length=1, nullable=True): def __repr__(self): return "BinaryType(%d, %s)" % (self.length, str(self._nullable).lower()) + def need_conversion(self): + return True + + def to_sql_type(self, obj): + return bytearray(obj) if obj is not None else obj + + def from_sql_type(self, obj): + return bytes(obj) if obj is not None else obj + class VarBinaryType(AtomicType): """ @@ -247,6 +256,37 @@ def __init__(self, length=1, nullable=True): def __repr__(self): return "VarBinaryType(%d, %s)" % (self.length, str(self._nullable).lower()) + def need_conversion(self): + return True + + def to_sql_type(self, obj): + return bytearray(obj) if obj is not None else obj + + def from_sql_type(self, obj): + return bytes(obj) if obj is not None else obj + + +class GeographyType(AtomicType): + """ + Geography data type. SQL GEOGRAPHY. + + The Python representation uses WKB bytes. + + :param nullable: boolean, whether the field can be null (None) or not. + """ + + def __init__(self, nullable=True): + super(GeographyType, self).__init__(nullable) + + def need_conversion(self): + return True + + def to_sql_type(self, obj): + return bytearray(obj) if obj is not None else obj + + def from_sql_type(self, obj): + return bytes(obj) if obj is not None else obj + class BooleanType(AtomicType): """ @@ -1385,6 +1425,7 @@ def deserialize(self, datum): int: BigIntType(), float: DoubleType(), str: VarCharType(0x7fffffff), + bytes: VarBinaryType(0x7fffffff), bytearray: VarBinaryType(0x7fffffff), decimal.Decimal: DecimalType(38, 18), datetime.date: DateType(), @@ -1700,6 +1741,8 @@ def _from_java_data_type(j_data_type): data_type = DataTypes.BINARY(logical_type.getLength(), logical_type.isNullable()) elif is_instance_of(logical_type, gateway.jvm.VarBinaryType): data_type = DataTypes.VARBINARY(logical_type.getLength(), logical_type.isNullable()) + elif is_instance_of(logical_type, gateway.jvm.GeographyType): + data_type = DataTypes.GEOGRAPHY(logical_type.isNullable()) elif is_instance_of(logical_type, gateway.jvm.DecimalType): data_type = DataTypes.DECIMAL(logical_type.getPrecision(), logical_type.getScale(), @@ -1850,6 +1893,8 @@ def _to_java_data_type(data_type: DataType): j_data_type = JDataTypes.VARBINARY(data_type.length) elif isinstance(data_type, BinaryType): j_data_type = JDataTypes.BINARY(data_type.length) + elif isinstance(data_type, GeographyType): + j_data_type = JDataTypes.GEOGRAPHY() elif isinstance(data_type, DecimalType): j_data_type = JDataTypes.DECIMAL(data_type.precision, data_type.scale) elif isinstance(data_type, DateType): @@ -1953,8 +1998,9 @@ def _to_java_data_type(data_type: DataType): DecimalType: (decimal.Decimal,), CharType: (str,), VarCharType: (str,), - BinaryType: (bytearray,), - VarBinaryType: (bytearray,), + BinaryType: (bytes, bytearray), + VarBinaryType: (bytes, bytearray), + GeographyType: (bytes, bytearray), DateType: (datetime.date, datetime.datetime), TimeType: (datetime.time,), TimestampType: (datetime.datetime,), @@ -2294,6 +2340,8 @@ def to_arrow_type(data_type: DataType): return pa.utf8() elif isinstance(data_type, BinaryType): return pa.binary(data_type.length) + elif isinstance(data_type, GeographyType): + return pa.binary() elif isinstance(data_type, VarBinaryType): return pa.binary() elif isinstance(data_type, DecimalType): @@ -2444,6 +2492,15 @@ def BYTES(nullable: bool = True) -> VarBinaryType: """ return DataTypes.VARBINARY(0x7fffffff, nullable) + @staticmethod + def GEOGRAPHY(nullable: bool = True) -> GeographyType: + """ + Data type of geography data represented as WKB bytes in Python. + + :param nullable: boolean, whether the type can be null (None) or not. + """ + return GeographyType(nullable) + @staticmethod def DECIMAL(precision: int, scale: int, nullable: bool = True) -> DecimalType: """ diff --git a/flink-python/src/main/java/org/apache/flink/api/common/python/PythonBridgeUtils.java b/flink-python/src/main/java/org/apache/flink/api/common/python/PythonBridgeUtils.java index 1205e79d2a8a83..9e10d18112125f 100644 --- a/flink-python/src/main/java/org/apache/flink/api/common/python/PythonBridgeUtils.java +++ b/flink-python/src/main/java/org/apache/flink/api/common/python/PythonBridgeUtils.java @@ -36,10 +36,12 @@ import org.apache.flink.core.memory.ByteArrayOutputStreamWithPos; import org.apache.flink.core.memory.DataOutputViewStreamWrapper; import org.apache.flink.streaming.api.typeinfo.python.PickledByteArrayTypeInfo; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.types.DataType; import org.apache.flink.table.types.logical.ArrayType; import org.apache.flink.table.types.logical.DateType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.LogicalType; import org.apache.flink.table.types.logical.MapType; import org.apache.flink.table.types.logical.RowType; @@ -193,6 +195,8 @@ private static Object getPickledBytesFromJavaObject(Object obj, LogicalType data } else { return pickler.dumps(obj); } + } else if (dataType instanceof GeographyType) { + return pickler.dumps(((GeographyData) obj).toBytes()); } else if (dataType instanceof RowType) { Row tmpRow = (Row) obj; LogicalType[] tmpRowFieldTypes = diff --git a/flink-python/src/main/java/org/apache/flink/table/runtime/arrow/ArrowUtils.java b/flink-python/src/main/java/org/apache/flink/table/runtime/arrow/ArrowUtils.java index d8b9dcdf772b66..8943502081213c 100644 --- a/flink-python/src/main/java/org/apache/flink/table/runtime/arrow/ArrowUtils.java +++ b/flink-python/src/main/java/org/apache/flink/table/runtime/arrow/ArrowUtils.java @@ -30,6 +30,7 @@ import org.apache.flink.table.api.internal.TableEnvironmentImpl; import org.apache.flink.table.api.internal.TableImpl; import org.apache.flink.table.data.ArrayData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.RowData; import org.apache.flink.table.data.columnar.vector.ColumnVector; import org.apache.flink.table.data.util.DataFormatConverters; @@ -83,6 +84,7 @@ import org.apache.flink.table.types.logical.DecimalType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LegacyTypeInformationType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; @@ -252,6 +254,9 @@ public static ArrowWriter createRowDataArrowWriter( private static ArrowFieldWriter createArrowFieldWriterForRow( ValueVector vector, LogicalType fieldType) { + if (fieldType instanceof GeographyType && vector instanceof VarBinaryVector) { + return createGeographyArrowWriterForRow((VarBinaryVector) vector); + } if (vector instanceof TinyIntVector) { return TinyIntWriter.forRow((TinyIntVector) vector); } else if (vector instanceof SmallIntVector) { @@ -329,6 +334,9 @@ private static ArrowFieldWriter createArrowFieldWriterForRow( private static ArrowFieldWriter createArrowFieldWriterForArray( ValueVector vector, LogicalType fieldType) { + if (fieldType instanceof GeographyType && vector instanceof VarBinaryVector) { + return createGeographyArrowWriterForArray((VarBinaryVector) vector); + } if (vector instanceof TinyIntVector) { return TinyIntWriter.forArray((TinyIntVector) vector); } else if (vector instanceof SmallIntVector) { @@ -771,6 +779,11 @@ public ArrowType visit(VarBinaryType varCharType) { return ArrowType.Binary.INSTANCE; } + @Override + public ArrowType visit(GeographyType geographyType) { + return ArrowType.Binary.INSTANCE; + } + @Override public ArrowType visit(DecimalType decimalType) { return new ArrowType.Decimal(decimalType.getPrecision(), decimalType.getScale()); @@ -874,4 +887,34 @@ private static int getPrecision(DecimalVector decimalVector) { } return precision; } + + private static ArrowFieldWriter createGeographyArrowWriterForRow( + VarBinaryVector vector) { + return new ArrowFieldWriter(vector) { + @Override + public void doWrite(RowData row, int ordinal) { + if (row.isNullAt(ordinal)) { + ((VarBinaryVector) getValueVector()).setNull(getCount()); + } else { + GeographyData geography = row.getGeography(ordinal); + ((VarBinaryVector) getValueVector()).setSafe(getCount(), geography.toBytes()); + } + } + }; + } + + private static ArrowFieldWriter createGeographyArrowWriterForArray( + VarBinaryVector vector) { + return new ArrowFieldWriter(vector) { + @Override + public void doWrite(ArrayData array, int ordinal) { + if (array.isNullAt(ordinal)) { + ((VarBinaryVector) getValueVector()).setNull(getCount()); + } else { + GeographyData geography = array.getGeography(ordinal); + ((VarBinaryVector) getValueVector()).setSafe(getCount(), geography.toBytes()); + } + } + }; + } } diff --git a/flink-python/src/main/java/org/apache/flink/table/runtime/typeutils/PythonTypeUtils.java b/flink-python/src/main/java/org/apache/flink/table/runtime/typeutils/PythonTypeUtils.java index 20278c60567f32..e5c4d163f488ec 100644 --- a/flink-python/src/main/java/org/apache/flink/table/runtime/typeutils/PythonTypeUtils.java +++ b/flink-python/src/main/java/org/apache/flink/table/runtime/typeutils/PythonTypeUtils.java @@ -31,6 +31,7 @@ import org.apache.flink.fnexecution.v1.FlinkFnApi; import org.apache.flink.table.data.ArrayData; import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.MapData; import org.apache.flink.table.data.RowData; import org.apache.flink.table.data.util.DataFormatConverters; @@ -47,6 +48,7 @@ import org.apache.flink.table.types.logical.DecimalType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LegacyTypeInformationType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; @@ -158,6 +160,11 @@ public TypeSerializer visit(VarBinaryType varBinaryType) { return BytePrimitiveArraySerializer.INSTANCE; } + @Override + public TypeSerializer visit(GeographyType geographyType) { + return GeographyTypeSerializer.INSTANCE; + } + @Override public TypeSerializer visit(RowType rowType) { final TypeSerializer[] fieldTypeSerializers = @@ -309,6 +316,14 @@ public FlinkFnApi.Schema.FieldType visit(VarBinaryType varBinaryType) { .build(); } + @Override + public FlinkFnApi.Schema.FieldType visit(GeographyType geographyType) { + return FlinkFnApi.Schema.FieldType.newBuilder() + .setTypeName(FlinkFnApi.Schema.TypeName.GEOGRAPHY) + .setNullable(geographyType.isNullable()) + .build(); + } + @Override public FlinkFnApi.Schema.FieldType visit(CharType charType) { return FlinkFnApi.Schema.FieldType.newBuilder() @@ -626,6 +641,27 @@ Time toExternalImpl(Integer value) { } } + /** GeographyData is exchanged with Python as ISO WKB bytes. */ + public static final class GeographyDataConverter + extends DataConverter { + + public static final GeographyDataConverter INSTANCE = new GeographyDataConverter(); + + private GeographyDataConverter() { + super(DataFormatConverters.GeographyConverter.INSTANCE); + } + + @Override + GeographyData toInternalImpl(byte[] value) { + return GeographyData.fromBytes(value); + } + + @Override + byte[] toExternalImpl(GeographyData value) { + return value.toBytes(); + } + } + /** * RowData will be converted to the Object Array [RowKind(as Long Object), Field Values(as * Object Array)]. @@ -830,6 +866,11 @@ public DataConverter visit(BinaryType binaryType) { return defaultConverter(binaryType); } + @Override + public DataConverter visit(GeographyType geographyType) { + return GeographyDataConverter.INSTANCE; + } + @Override public DataConverter visit(DateType dateType) { return new IdentityDataConverter<>(DataFormatConverters.DateConverter.INSTANCE); diff --git a/flink-python/src/main/java/org/apache/flink/table/utils/python/PythonTableUtils.java b/flink-python/src/main/java/org/apache/flink/table/utils/python/PythonTableUtils.java index 01dfab186dddcf..5f9bbecae408ff 100644 --- a/flink-python/src/main/java/org/apache/flink/table/utils/python/PythonTableUtils.java +++ b/flink-python/src/main/java/org/apache/flink/table/utils/python/PythonTableUtils.java @@ -31,6 +31,7 @@ import org.apache.flink.table.data.GenericArrayData; import org.apache.flink.table.data.GenericMapData; import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.RowData; import org.apache.flink.table.data.StringData; import org.apache.flink.table.data.TimestampData; @@ -46,6 +47,7 @@ import org.apache.flink.table.types.logical.DecimalType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; @@ -393,6 +395,10 @@ private static Function converter(LogicalType logicalType) { }; } + if (logicalType instanceof GeographyType) { + return c -> c instanceof byte[] ? GeographyData.fromBytes((byte[]) c) : null; + } + if (logicalType instanceof ArrayType) { LogicalType elementType = ((ArrayType) logicalType).getElementType(); Function elementConverter = converter(elementType); diff --git a/flink-python/src/test/java/org/apache/flink/table/runtime/typeutils/PythonTypeUtilsTest.java b/flink-python/src/test/java/org/apache/flink/table/runtime/typeutils/PythonTypeUtilsTest.java index c061d2d3ce032f..a1e372d3b58d95 100644 --- a/flink-python/src/test/java/org/apache/flink/table/runtime/typeutils/PythonTypeUtilsTest.java +++ b/flink-python/src/test/java/org/apache/flink/table/runtime/typeutils/PythonTypeUtilsTest.java @@ -22,7 +22,9 @@ import org.apache.flink.fnexecution.v1.FlinkFnApi; import org.apache.flink.table.catalog.UnresolvedIdentifier; import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.types.logical.BigIntType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LogicalType; import org.apache.flink.table.types.logical.RowType; @@ -39,6 +41,11 @@ /** Tests for {@link PythonTypeUtils}. */ class PythonTypeUtilsTest { + private static final byte[] POINT_WKB = + new byte[] { + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, (byte) 0xF0, 0x3F, 0, 0, 0, 0, 0, 0, 0, 0x40 + }; + @Test void testLogicalTypetoInternalSerializer() { List rowFields = new ArrayList<>(); @@ -75,6 +82,29 @@ void testLogicalTypeToDataConverter() { assertThat(externalData).isEqualTo(10L); } + @Test + void testGeographyLogicalTypeToProtoType() { + FlinkFnApi.Schema.FieldType protoType = PythonTypeUtils.toProtoType(new GeographyType()); + + assertThat(protoType.getTypeName()).isEqualTo(FlinkFnApi.Schema.TypeName.GEOGRAPHY); + assertThat(protoType.getNullable()).isTrue(); + } + + @Test + void testGeographyLogicalTypeToDataConverter() { + PythonTypeUtils.DataConverter converter = + PythonTypeUtils.toDataConverter(new GeographyType()); + + GenericRowData data = new GenericRowData(1); + data.setField(0, GeographyData.fromBytes(POINT_WKB)); + + byte[] externalData = (byte[]) converter.toExternal(data, 0); + assertThat(externalData).isEqualTo(POINT_WKB); + + GeographyData internalData = (GeographyData) converter.toInternal(externalData); + assertThat(internalData.toBytes()).isEqualTo(POINT_WKB); + } + @Test void testUnsupportedTypeSerializer() { LogicalType logicalType = diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/operators/AbstractAsyncRunnableStreamOperatorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/operators/AbstractAsyncRunnableStreamOperatorTest.java index b073f6cbed1210..1601b06bda9986 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/operators/AbstractAsyncRunnableStreamOperatorTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/operators/AbstractAsyncRunnableStreamOperatorTest.java @@ -253,8 +253,11 @@ void testCheckpointDrain() throws Exception { }); ((AbstractAsyncRunnableStreamOperator) testHarness.getOperator()) .postProcessElement(); - assertThat(asyncExecutionController.getInFlightRecordNum()).isEqualTo(1); - unblockAsyncRequest.complete(null); + try { + assertThat(asyncExecutionController.getInFlightRecordNum()).isEqualTo(1); + } finally { + unblockAsyncRequest.complete(null); + } testHarness.drainAsyncRequests(); assertThat(asyncExecutionController.getInFlightRecordNum()).isEqualTo(0); } diff --git a/flink-table/flink-sql-parser/src/main/codegen/data/Parser.tdd b/flink-table/flink-sql-parser/src/main/codegen/data/Parser.tdd index b7e6d7ab9ea3f4..db08bbfbf2c329 100644 --- a/flink-table/flink-sql-parser/src/main/codegen/data/Parser.tdd +++ b/flink-table/flink-sql-parser/src/main/codegen/data/Parser.tdd @@ -173,6 +173,7 @@ "org.apache.flink.sql.parser.type.ExtendedSqlCollectionTypeNameSpec" "org.apache.flink.sql.parser.type.ExtendedSqlRowTypeNameSpec" "org.apache.flink.sql.parser.type.SqlBitmapTypeNameSpec" + "org.apache.flink.sql.parser.type.SqlGeographyTypeNameSpec" "org.apache.flink.sql.parser.type.SqlRawTypeNameSpec" "org.apache.flink.sql.parser.type.SqlStructuredTypeNameSpec" "org.apache.flink.sql.parser.type.SqlTimestampLtzTypeNameSpec" @@ -225,6 +226,7 @@ "FROM_TIMESTAMP" "FUNCTIONS" "FRESHNESS" + "GEOGRAPHY" "HASH" "IF" "JSON_EXECUTION_PLAN" @@ -719,6 +721,7 @@ "ExtendedSqlRowTypeName()" "SqlStructuredTypeName()" "SqlBitmapTypeName()" + "SqlGeographyTypeName()" ] # List of methods for parsing builtin function calls. diff --git a/flink-table/flink-sql-parser/src/main/codegen/includes/parserImpls.ftl b/flink-table/flink-sql-parser/src/main/codegen/includes/parserImpls.ftl index 2b8250ea98ed9e..ede54c27ca86e0 100644 --- a/flink-table/flink-sql-parser/src/main/codegen/includes/parserImpls.ftl +++ b/flink-table/flink-sql-parser/src/main/codegen/includes/parserImpls.ftl @@ -2666,6 +2666,17 @@ SqlTypeNameSpec SqlBitmapTypeName() : } } +/** Parses GEOGRAPHY type. */ +SqlTypeNameSpec SqlGeographyTypeName() : +{ +} +{ + + { + return new SqlGeographyTypeNameSpec(getPos()); + } +} + /** * Parse a "name1 type1 [ NULL | NOT NULL] [ comment ] * [, name2 type2 [ NULL | NOT NULL] [ comment ] ]* ..." list. diff --git a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/SqlGeographyTypeNameSpec.java b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/SqlGeographyTypeNameSpec.java new file mode 100644 index 00000000000000..55a3d7ff4fd7d6 --- /dev/null +++ b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/SqlGeographyTypeNameSpec.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.sql.parser.type; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.calcite.ExtendedRelTypeFactory; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlTypeNameSpec; +import org.apache.calcite.sql.SqlWriter; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.sql.validate.SqlValidator; +import org.apache.calcite.util.Litmus; + +/** Represents the GEOGRAPHY data type. */ +@Internal +public final class SqlGeographyTypeNameSpec extends SqlTypeNameSpec { + + private static final String GEOGRAPHY_TYPE_NAME = "GEOGRAPHY"; + + public SqlGeographyTypeNameSpec(SqlParserPos pos) { + super(new SqlIdentifier(GEOGRAPHY_TYPE_NAME, pos), pos); + } + + @Override + public RelDataType deriveType(SqlValidator validator) { + return ((ExtendedRelTypeFactory) validator.getTypeFactory()).createGeographyType(); + } + + @Override + public void unparse(SqlWriter writer, int leftPrec, int rightPrec) { + writer.keyword(GEOGRAPHY_TYPE_NAME); + } + + @Override + public boolean equalsDeep(SqlTypeNameSpec spec, Litmus litmus) { + if (!(spec instanceof SqlGeographyTypeNameSpec)) { + return litmus.fail("{} != {}", this, spec); + } + return litmus.succeed(); + } +} diff --git a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/table/calcite/ExtendedRelTypeFactory.java b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/table/calcite/ExtendedRelTypeFactory.java index 57b9bc615ec930..57be54e4cb6bd4 100644 --- a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/table/calcite/ExtendedRelTypeFactory.java +++ b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/table/calcite/ExtendedRelTypeFactory.java @@ -44,4 +44,7 @@ RelDataType createStructuredType( /** Creates a BITMAP type. */ RelDataType createBitmapType(); + + /** Creates a GEOGRAPHY type. */ + RelDataType createGeographyType(); } diff --git a/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/FlinkSqlParserImplTest.java b/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/FlinkSqlParserImplTest.java index 92f8e34c770a00..ee33ea2e37daf2 100644 --- a/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/FlinkSqlParserImplTest.java +++ b/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/FlinkSqlParserImplTest.java @@ -4074,4 +4074,23 @@ void testBitmapType() { sql("CREATE TABLE t (\n" + "^bitmap^ INT" + "\n)") .fails("(?s).*Encountered \"bitmap\" at line 2, column 1.\n.*"); } + + @Test + void testGeographyType() { + sql("CREATE TABLE t (\n" + "g geography" + "\n)") + .ok("CREATE TABLE `T` (\n" + " `G` GEOGRAPHY\n" + ")"); + + sql("CREATE TABLE t (\n" + "g geography NOT NULL" + "\n)") + .ok("CREATE TABLE `T` (\n" + " `G` GEOGRAPHY NOT NULL\n" + ")"); + + // GEOGRAPHY takes no parameters + sql("CREATE TABLE t (\n" + "g geography^(^1)" + "\n)") + .fails("(?s).*Encountered \"\\(\" at line 2, column 12.\n.*"); + sql("CREATE TABLE t (\n" + "g geography^(^4326)" + "\n)") + .fails("(?s).*Encountered \"\\(\" at line 2, column 12.\n.*"); + + // GEOGRAPHY is a reserved keyword and cannot be used as an identifier without escaping + sql("CREATE TABLE t (\n" + "^geography^ INT" + "\n)") + .fails("(?s).*Encountered \"geography\" at line 2, column 1.\n.*"); + } } diff --git a/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/TestRelDataTypeFactory.java b/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/TestRelDataTypeFactory.java index 34fe9926079b3c..070c77905a335f 100644 --- a/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/TestRelDataTypeFactory.java +++ b/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/TestRelDataTypeFactory.java @@ -53,6 +53,11 @@ public RelDataType createBitmapType() { return canonize(new DummyBitmapType()); } + @Override + public RelDataType createGeographyType() { + return canonize(new DummyGeographyType()); + } + private static class DummyRawType extends RelDataTypeImpl { private final String className; @@ -117,4 +122,16 @@ protected void generateTypeString(StringBuilder sb, boolean withDetail) { sb.append("BITMAP"); } } + + private static class DummyGeographyType extends RelDataTypeImpl { + + DummyGeographyType() { + computeDigest(); + } + + @Override + protected void generateTypeString(StringBuilder sb, boolean withDetail) { + sb.append("GEOGRAPHY"); + } + } } diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/api/DataTypes.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/api/DataTypes.java index af1e718d90e04c..3912c178328526 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/api/DataTypes.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/api/DataTypes.java @@ -44,6 +44,7 @@ import org.apache.flink.table.types.logical.DescriptorType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; @@ -1075,6 +1076,15 @@ public static DataType BITMAP() { return new AtomicDataType(new BitmapType()); } + /** + * Data type of geography data. + * + * @see GeographyType + */ + public static DataType GEOGRAPHY() { + return new AtomicDataType(new GeographyType()); + } + // -------------------------------------------------------------------------------------------- // Helper functions // -------------------------------------------------------------------------------------------- diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/ArrayData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/ArrayData.java index 811c9be70e6b87..728d3c228f05da 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/ArrayData.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/ArrayData.java @@ -123,6 +123,12 @@ default Bitmap getBitmap(int pos) { "This ArrayData implementation does not support Bitmap type."); } + /** Returns the geography value at the given position. */ + default GeographyData getGeography(int pos) { + throw new UnsupportedOperationException( + "This ArrayData implementation does not support Geography type."); + } + // ------------------------------------------------------------------------------------------ // Conversion Utilities // ------------------------------------------------------------------------------------------ @@ -225,6 +231,9 @@ static ElementGetter createElementGetter(LogicalType elementType) { case BITMAP: elementGetter = ArrayData::getBitmap; break; + case GEOGRAPHY: + elementGetter = ArrayData::getGeography; + break; case NULL: case SYMBOL: case UNRESOLVED: diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GenericArrayData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GenericArrayData.java index f609c2f22f1311..08856cdb0ceb1e 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GenericArrayData.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GenericArrayData.java @@ -265,6 +265,11 @@ public Bitmap getBitmap(int pos) { return (Bitmap) getObject(pos); } + @Override + public GeographyData getGeography(int pos) { + return (GeographyData) getObject(pos); + } + private Object getObject(int pos) { return ((Object[]) array)[pos]; } diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GenericRowData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GenericRowData.java index b234edef62d05b..98c77bca28f06d 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GenericRowData.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GenericRowData.java @@ -192,6 +192,11 @@ public byte[] getBinary(int pos) { return (byte[]) this.fields[pos]; } + @Override + public GeographyData getGeography(int pos) { + return (GeographyData) this.fields[pos]; + } + @Override public ArrayData getArray(int pos) { return (ArrayData) this.fields[pos]; diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GeographyData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GeographyData.java new file mode 100644 index 00000000000000..f65075be637c80 --- /dev/null +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GeographyData.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.data; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.table.data.binary.BinaryGeographyData; +import org.apache.flink.table.types.logical.GeographyType; + +/** An internal data structure representing data of {@link GeographyType}. */ +@PublicEvolving +public interface GeographyData { + + /** ISO WKB subtype ID for Point geometries. */ + int POINT = 1; + + /** ISO WKB subtype ID for LineString geometries. */ + int LINE_STRING = 2; + + /** ISO WKB subtype ID for Polygon geometries. */ + int POLYGON = 3; + + /** ISO WKB subtype ID for MultiPoint geometries. */ + int MULTI_POINT = 4; + + /** ISO WKB subtype ID for MultiLineString geometries. */ + int MULTI_LINE_STRING = 5; + + /** ISO WKB subtype ID for MultiPolygon geometries. */ + int MULTI_POLYGON = 6; + + /** ISO WKB subtype ID for GeometryCollection geometries. */ + int GEOMETRY_COLLECTION = 7; + + /** + * Converts this {@link GeographyData} object to an ISO WKB byte array. + * + *

Note: The returned byte array may be reused. + */ + byte[] toBytes(); + + /** Returns the ISO WKB subtype ID. */ + int subtypeId(); + + /** Returns the size in bytes of the ISO WKB payload. */ + int sizeInBytes(); + + // ------------------------------------------------------------------------------------------ + // Construction Utilities + // ------------------------------------------------------------------------------------------ + + /** + * Creates an instance of {@link GeographyData} from the given ISO WKB byte array. Returns + * {@code null} if the input is {@code null}. + */ + static GeographyData fromBytes(byte[] bytes) { + return BinaryGeographyData.fromBytes(bytes); + } + + /** + * Creates an instance of {@link GeographyData} from the given ISO WKB byte range. Returns + * {@code null} if the input is {@code null}. + */ + static GeographyData fromBytes(byte[] bytes, int offset, int numBytes) { + return BinaryGeographyData.fromBytes(bytes, offset, numBytes); + } +} diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/RowData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/RowData.java index ca43f1608f99ef..a783ecb572a698 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/RowData.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/RowData.java @@ -111,6 +111,8 @@ * +--------------------------------+-----------------------------------------+ * | BITMAP | {@link Bitmap} | * +--------------------------------+-----------------------------------------+ + * | GEOGRAPHY | {@link GeographyData} | + * +--------------------------------+-----------------------------------------+ * * *

Nullability is always handled by the container data structure. @@ -214,6 +216,12 @@ default Bitmap getBitmap(int pos) { "This RowData implementation does not support Bitmap type."); } + /** Returns the geography value at the given position. */ + default GeographyData getGeography(int pos) { + throw new UnsupportedOperationException( + "This RowData implementation does not support Geography type."); + } + // ------------------------------------------------------------------------------------------ // Access Utilities // ------------------------------------------------------------------------------------------ @@ -299,6 +307,9 @@ static FieldGetter createFieldGetter(LogicalType fieldType, int fieldPos) { case BITMAP: fieldGetter = row -> row.getBitmap(fieldPos); break; + case GEOGRAPHY: + fieldGetter = row -> row.getGeography(fieldPos); + break; case NULL: case SYMBOL: case UNRESOLVED: diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryArrayData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryArrayData.java index 10a8b3e6ef71c7..f09f07d25996c1 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryArrayData.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryArrayData.java @@ -23,6 +23,7 @@ import org.apache.flink.core.memory.MemorySegmentFactory; import org.apache.flink.table.data.ArrayData; import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.MapData; import org.apache.flink.table.data.RawValueData; import org.apache.flink.table.data.RowData; @@ -95,6 +96,7 @@ public static int calculateFixLengthPartSize(LogicalType type) { case RAW: case VARIANT: case BITMAP: + case GEOGRAPHY: // long and double are 8 bytes; // otherwise it stores the length and offset of the variable-length part for types // such as is string, map, etc. @@ -269,6 +271,14 @@ public byte[] getBinary(int pos) { return BinarySegmentUtils.readBinary(segments, offset, fieldOffset, offsetAndSize); } + @Override + public GeographyData getGeography(int pos) { + assertIndexIsValid(pos); + int fieldOffset = getElementOffset(pos, 8); + final long offsetAndSize = BinarySegmentUtils.getLong(segments, fieldOffset); + return BinarySegmentUtils.readGeographyData(segments, offset, fieldOffset, offsetAndSize); + } + @Override public ArrayData getArray(int pos) { assertIndexIsValid(pos); diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryGeographyData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryGeographyData.java new file mode 100644 index 00000000000000..4256f786a86816 --- /dev/null +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryGeographyData.java @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.data.binary; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.core.memory.MemorySegment; +import org.apache.flink.core.memory.MemorySegmentFactory; +import org.apache.flink.table.api.TableRuntimeException; +import org.apache.flink.table.data.GeographyData; + +import java.util.Arrays; + +/** + * A binary implementation of {@link GeographyData} backed by raw ISO WKB bytes. + * + *

GEOGRAPHY uses OGC:CRS84 by contract, but ISO WKB does not encode CRS or SRID metadata. This + * container stores the raw ISO WKB payload only; CRS validation, CRS transformation, and EWKB/SRID + * handling belong to constructors, functions, and connector schema mapping. + */ +@Internal +public final class BinaryGeographyData extends BinarySection implements GeographyData { + + private static final int MIN_WKB_HEADER_SIZE = 5; + private static final int WKB_COUNT_SIZE = 4; + private static final int WKB_POINT_COORDINATE_SIZE = 16; + private static final int BIG_ENDIAN = 0; + private static final int LITTLE_ENDIAN = 1; + + private final int subtypeId; + + private BinaryGeographyData(MemorySegment[] segments, int offset, int sizeInBytes) { + super(segments, offset, sizeInBytes); + this.subtypeId = readSubtypeId(segments, offset, sizeInBytes); + } + + /** Creates a {@link BinaryGeographyData} instance from the given address and length. */ + public static BinaryGeographyData fromAddress( + MemorySegment[] segments, int offset, int numBytes) { + return new BinaryGeographyData(segments, offset, numBytes); + } + + /** Creates a {@link BinaryGeographyData} instance from the given ISO WKB bytes. */ + public static BinaryGeographyData fromBytes(byte[] bytes) { + return bytes == null ? null : fromBytes(bytes, 0, bytes.length); + } + + /** Creates a {@link BinaryGeographyData} instance from the given ISO WKB byte range. */ + public static BinaryGeographyData fromBytes(byte[] bytes, int offset, int numBytes) { + if (bytes == null) { + return null; + } + checkRange(bytes, offset, numBytes); + byte[] copy = Arrays.copyOfRange(bytes, offset, offset + numBytes); + return fromAddress(new MemorySegment[] {MemorySegmentFactory.wrap(copy)}, 0, copy.length); + } + + @Override + public byte[] toBytes() { + return BinarySegmentUtils.copyToBytes(segments, offset, sizeInBytes); + } + + @Override + public int subtypeId() { + return subtypeId; + } + + @Override + public int sizeInBytes() { + return sizeInBytes; + } + + private static void checkRange(byte[] bytes, int offset, int numBytes) { + if (offset < 0 || numBytes < 0 || offset > bytes.length - numBytes) { + throw new TableRuntimeException( + String.format( + "Invalid ISO WKB byte range: offset %d, length %d, array length %d.", + offset, numBytes, bytes.length)); + } + } + + private static int readSubtypeId(MemorySegment[] segments, int offset, int sizeInBytes) { + final long endOffset = (long) offset + sizeInBytes; + final GeometryHeader header = readHeader(segments, offset, endOffset); + final long consumedOffset = validateGeometry(segments, offset, endOffset); + if (consumedOffset != endOffset) { + throw new TableRuntimeException( + String.format( + "Malformed ISO WKB payload. Found %d trailing byte(s).", + endOffset - consumedOffset)); + } + return header.subtypeId; + } + + private static long validateGeometry( + MemorySegment[] segments, long geometryOffset, long endOffset) { + final GeometryHeader header = readHeader(segments, geometryOffset, endOffset); + long cursor = geometryOffset + MIN_WKB_HEADER_SIZE; + + switch (header.subtypeId) { + case GeographyData.POINT: + return requireBytes( + cursor, WKB_POINT_COORDINATE_SIZE, endOffset, "POINT coordinates") + + WKB_POINT_COORDINATE_SIZE; + case GeographyData.LINE_STRING: + return skipCoordinateSequence(segments, cursor, endOffset, header.byteOrder); + case GeographyData.POLYGON: + return skipPolygon(segments, cursor, endOffset, header.byteOrder); + case GeographyData.MULTI_POINT: + return skipTypedGeometryCollection( + segments, cursor, endOffset, header.byteOrder, GeographyData.POINT); + case GeographyData.MULTI_LINE_STRING: + return skipTypedGeometryCollection( + segments, cursor, endOffset, header.byteOrder, GeographyData.LINE_STRING); + case GeographyData.MULTI_POLYGON: + return skipTypedGeometryCollection( + segments, cursor, endOffset, header.byteOrder, GeographyData.POLYGON); + case GeographyData.GEOMETRY_COLLECTION: + return skipGeometryCollection(segments, cursor, endOffset, header.byteOrder); + default: + throw new TableRuntimeException( + String.format( + "Malformed ISO WKB payload. Unsupported geography subtype ID %d.", + header.subtypeId)); + } + } + + private static long skipCoordinateSequence( + MemorySegment[] segments, long offset, long endOffset, int byteOrder) { + final long numPoints = + readUnsignedInt(segments, offset, endOffset, byteOrder, "point count"); + long cursor = offset + WKB_COUNT_SIZE; + final long coordinateSequenceSize = + multiplyExact(numPoints, WKB_POINT_COORDINATE_SIZE, "coordinate sequence"); + return requireBytes(cursor, coordinateSequenceSize, endOffset, "coordinate sequence") + + coordinateSequenceSize; + } + + private static long skipPolygon( + MemorySegment[] segments, long offset, long endOffset, int byteOrder) { + final long numRings = readUnsignedInt(segments, offset, endOffset, byteOrder, "ring count"); + long cursor = offset + WKB_COUNT_SIZE; + for (long i = 0; i < numRings; i++) { + cursor = skipCoordinateSequence(segments, cursor, endOffset, byteOrder); + } + return cursor; + } + + private static long skipTypedGeometryCollection( + MemorySegment[] segments, + long offset, + long endOffset, + int byteOrder, + int expectedSubtypeId) { + final long numGeometries = + readUnsignedInt(segments, offset, endOffset, byteOrder, "geometry count"); + long cursor = offset + WKB_COUNT_SIZE; + for (long i = 0; i < numGeometries; i++) { + final GeometryHeader nestedHeader = readHeader(segments, cursor, endOffset); + if (nestedHeader.subtypeId != expectedSubtypeId) { + throw new TableRuntimeException( + String.format( + "Malformed ISO WKB payload. Expected nested subtype ID %d but found %d.", + expectedSubtypeId, nestedHeader.subtypeId)); + } + cursor = validateGeometry(segments, cursor, endOffset); + } + return cursor; + } + + private static long skipGeometryCollection( + MemorySegment[] segments, long offset, long endOffset, int byteOrder) { + final long numGeometries = + readUnsignedInt(segments, offset, endOffset, byteOrder, "geometry count"); + long cursor = offset + WKB_COUNT_SIZE; + for (long i = 0; i < numGeometries; i++) { + cursor = validateGeometry(segments, cursor, endOffset); + } + return cursor; + } + + private static GeometryHeader readHeader( + MemorySegment[] segments, long offset, long endOffset) { + requireBytes(offset, MIN_WKB_HEADER_SIZE, endOffset, "WKB header"); + + final int byteOrder = BinarySegmentUtils.getByte(segments, (int) offset) & 0xFF; + if (byteOrder != BIG_ENDIAN && byteOrder != LITTLE_ENDIAN) { + throw new TableRuntimeException( + String.format( + "Malformed ISO WKB payload. Unsupported byte order %d.", byteOrder)); + } + + final long subtypeId = + readUnsignedInt(segments, offset + 1, endOffset, byteOrder, "subtype ID"); + + if (subtypeId < GeographyData.POINT || subtypeId > GeographyData.GEOMETRY_COLLECTION) { + throw new TableRuntimeException( + String.format( + "Malformed ISO WKB payload. Unsupported geography subtype ID %d.", + subtypeId)); + } + return new GeometryHeader(byteOrder, (int) subtypeId); + } + + private static long readUnsignedInt( + MemorySegment[] segments, + long offset, + long endOffset, + int byteOrder, + String fieldName) { + requireBytes(offset, WKB_COUNT_SIZE, endOffset, fieldName); + + if (byteOrder == LITTLE_ENDIAN) { + return (BinarySegmentUtils.getByte(segments, (int) offset) & 0xFFL) + | ((BinarySegmentUtils.getByte(segments, (int) offset + 1) & 0xFFL) << 8) + | ((BinarySegmentUtils.getByte(segments, (int) offset + 2) & 0xFFL) << 16) + | ((BinarySegmentUtils.getByte(segments, (int) offset + 3) & 0xFFL) << 24); + } + return ((BinarySegmentUtils.getByte(segments, (int) offset) & 0xFFL) << 24) + | ((BinarySegmentUtils.getByte(segments, (int) offset + 1) & 0xFFL) << 16) + | ((BinarySegmentUtils.getByte(segments, (int) offset + 2) & 0xFFL) << 8) + | (BinarySegmentUtils.getByte(segments, (int) offset + 3) & 0xFFL); + } + + private static long requireBytes( + long offset, long numBytes, long endOffset, String componentName) { + if (offset > endOffset || endOffset - offset < numBytes) { + throw new TableRuntimeException( + String.format( + "Malformed ISO WKB payload. Incomplete %s: expected %d byte(s) but found %d.", + componentName, numBytes, Math.max(0, endOffset - offset))); + } + return offset; + } + + private static long multiplyExact(long value, int factor, String componentName) { + final long result = value * factor; + if (value != 0 && result / value != factor) { + throw new TableRuntimeException( + String.format("Malformed ISO WKB payload. %s size overflows.", componentName)); + } + return result; + } + + private static final class GeometryHeader { + private final int byteOrder; + private final int subtypeId; + + private GeometryHeader(int byteOrder, int subtypeId) { + this.byteOrder = byteOrder; + this.subtypeId = subtypeId; + } + } +} diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryRowData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryRowData.java index 45bcf7cd3f1128..04c9a0e50ed43d 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryRowData.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryRowData.java @@ -22,6 +22,7 @@ import org.apache.flink.core.memory.MemorySegmentFactory; import org.apache.flink.table.data.ArrayData; import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.MapData; import org.apache.flink.table.data.RawValueData; import org.apache.flink.table.data.RowData; @@ -372,6 +373,14 @@ public byte[] getBinary(int pos) { return BinarySegmentUtils.readBinary(segments, offset, fieldOffset, offsetAndLen); } + @Override + public GeographyData getGeography(int pos) { + assertIndexIsValid(pos); + int fieldOffset = getFieldOffset(pos); + final long offsetAndLen = segments[0].getLong(fieldOffset); + return BinarySegmentUtils.readGeographyData(segments, offset, fieldOffset, offsetAndLen); + } + @Override public ArrayData getArray(int pos) { assertIndexIsValid(pos); diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinarySegmentUtils.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinarySegmentUtils.java index 653fc559df9bce..2dd12978d6bf5f 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinarySegmentUtils.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinarySegmentUtils.java @@ -1056,6 +1056,34 @@ public static byte[] readBinary( } } + /** + * Get geography data, if len less than 8, it will be included in variablePartOffsetAndLen. + * + * @param baseOffset base offset of composite binary format. + * @param fieldOffset absolute start offset of 'variablePartOffsetAndLen'. + * @param variablePartOffsetAndLen a long value, real data or offset and len. + */ + public static BinaryGeographyData readGeographyData( + MemorySegment[] segments, + int baseOffset, + int fieldOffset, + long variablePartOffsetAndLen) { + long mark = variablePartOffsetAndLen & HIGHEST_FIRST_BIT; + if (mark == 0) { + final int subOffset = (int) (variablePartOffsetAndLen >> 32); + final int len = (int) variablePartOffsetAndLen; + return BinaryGeographyData.fromAddress(segments, baseOffset + subOffset, len); + } else { + int len = (int) ((variablePartOffsetAndLen & HIGHEST_SECOND_TO_EIGHTH_BIT) >>> 56); + if (BinarySegmentUtils.LITTLE_ENDIAN) { + return BinaryGeographyData.fromAddress(segments, fieldOffset, len); + } else { + // fieldOffset + 1 to skip header. + return BinaryGeographyData.fromAddress(segments, fieldOffset + 1, len); + } + } + } + /** * Get binary string, if len less than 8, will be include in variablePartOffsetAndLen. * diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/NestedRowData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/NestedRowData.java index 27085b487ae50d..225d3e103fe103 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/NestedRowData.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/NestedRowData.java @@ -21,6 +21,7 @@ import org.apache.flink.core.memory.MemorySegmentFactory; import org.apache.flink.table.data.ArrayData; import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.MapData; import org.apache.flink.table.data.RawValueData; import org.apache.flink.table.data.RowData; @@ -294,6 +295,14 @@ public byte[] getBinary(int pos) { return BinarySegmentUtils.readBinary(segments, offset, fieldOffset, offsetAndLen); } + @Override + public GeographyData getGeography(int pos) { + assertIndexIsValid(pos); + int fieldOffset = getFieldOffset(pos); + final long offsetAndLen = BinarySegmentUtils.getLong(segments, fieldOffset); + return BinarySegmentUtils.readGeographyData(segments, offset, fieldOffset, offsetAndLen); + } + @Override public RowData getRow(int pos, int numFields) { assertIndexIsValid(pos); diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/columnar/ColumnarArrayData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/columnar/ColumnarArrayData.java index a3b8e7dd56daf9..9c6e6ad8c317d0 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/columnar/ColumnarArrayData.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/columnar/ColumnarArrayData.java @@ -21,6 +21,7 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.table.data.ArrayData; import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.MapData; import org.apache.flink.table.data.RawValueData; import org.apache.flink.table.data.RowData; @@ -147,6 +148,12 @@ public byte[] getBinary(int pos) { } } + @Override + public GeographyData getGeography(int pos) { + BytesColumnVector.Bytes byteArray = getByteArray(pos); + return GeographyData.fromBytes(byteArray.data, byteArray.offset, byteArray.len); + } + @Override public ArrayData getArray(int pos) { return ((ArrayColumnVector) data).getArray(offset + pos); diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/columnar/ColumnarRowData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/columnar/ColumnarRowData.java index bd3dce1edf6bae..0413ac980f737a 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/columnar/ColumnarRowData.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/columnar/ColumnarRowData.java @@ -21,6 +21,7 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.table.data.ArrayData; import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.MapData; import org.apache.flink.table.data.RawValueData; import org.apache.flink.table.data.RowData; @@ -152,6 +153,12 @@ public byte[] getBinary(int pos) { } } + @Override + public GeographyData getGeography(int pos) { + Bytes byteArray = vectorizedColumnBatch.getByteArray(rowId, pos); + return GeographyData.fromBytes(byteArray.data, byteArray.offset, byteArray.len); + } + @Override public RowData getRow(int pos, int numFields) { return vectorizedColumnBatch.getRow(rowId, pos); diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/utils/JoinedRowData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/utils/JoinedRowData.java index b9eff05a6c388f..d1420debfdb07a 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/utils/JoinedRowData.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/utils/JoinedRowData.java @@ -21,6 +21,7 @@ import org.apache.flink.annotation.PublicEvolving; import org.apache.flink.table.data.ArrayData; import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.MapData; import org.apache.flink.table.data.RawValueData; import org.apache.flink.table.data.RowData; @@ -269,6 +270,15 @@ public Bitmap getBitmap(int pos) { } } + @Override + public GeographyData getGeography(int pos) { + if (pos < row1.getArity()) { + return row1.getGeography(pos); + } else { + return row2.getGeography(pos - row1.getArity()); + } + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/utils/ProjectedRowData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/utils/ProjectedRowData.java index 7f29ca5e613236..e6a173bb4cd991 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/utils/ProjectedRowData.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/utils/ProjectedRowData.java @@ -22,6 +22,7 @@ import org.apache.flink.table.connector.Projection; import org.apache.flink.table.data.ArrayData; import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.MapData; import org.apache.flink.table.data.RawValueData; import org.apache.flink.table.data.RowData; @@ -178,6 +179,11 @@ public Bitmap getBitmap(int pos) { return row.getBitmap(indexMapping[pos]); } + @Override + public GeographyData getGeography(int pos) { + return row.getGeography(indexMapping[pos]); + } + @Override public boolean equals(Object o) { throw new UnsupportedOperationException("Projected row data cannot be compared"); diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/expressions/ValueLiteralExpression.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/expressions/ValueLiteralExpression.java index 72a2534b64fdb1..4427b1be21993b 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/expressions/ValueLiteralExpression.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/expressions/ValueLiteralExpression.java @@ -21,6 +21,7 @@ import org.apache.flink.annotation.PublicEvolving; import org.apache.flink.table.api.TableException; import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.types.DataType; import org.apache.flink.table.types.inference.CallContext; import org.apache.flink.table.types.logical.DayTimeIntervalType; @@ -273,6 +274,11 @@ public String asSerializableString(SqlFactory sqlFactory) { case BINARY: case VARBINARY: return String.format("X'%s'", StringUtils.byteToHexString((byte[]) value)); + case GEOGRAPHY: + return String.format( + "ST_GEOGFROMWKB(X'%s')", + StringUtils.byteToHexString( + getValueAs(GeographyData.class).get().toBytes())); case DATE: return String.format("DATE '%s'", getValueAs(LocalDate.class).get()); case TIME_WITHOUT_TIME_ZONE: diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java index dae5f976da5fd0..b95f6f6910688d 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java @@ -3116,6 +3116,63 @@ ANY, and(logical(LogicalTypeRoot.BOOLEAN), LITERAL) "org.apache.flink.table.runtime.functions.scalar.TryParseJsonFunction") .build(); + // -------------------------------------------------------------------------------------------- + // Geography functions + // -------------------------------------------------------------------------------------------- + + public static final BuiltInFunctionDefinition ST_GEOGFROMTEXT = + BuiltInFunctionDefinition.newBuilder() + .name("ST_GEOGFROMTEXT") + .kind(SCALAR) + .inputTypeStrategy( + sequence( + Collections.singletonList("text"), + Collections.singletonList( + logical(LogicalTypeFamily.CHARACTER_STRING)))) + .outputTypeStrategy(nullableIfArgs(explicit(DataTypes.GEOGRAPHY()))) + .runtimeClass( + "org.apache.flink.table.runtime.functions.scalar.StGeogFromTextFunction") + .build(); + + public static final BuiltInFunctionDefinition ST_GEOGFROMWKB = + BuiltInFunctionDefinition.newBuilder() + .name("ST_GEOGFROMWKB") + .kind(SCALAR) + .inputTypeStrategy( + sequence( + Collections.singletonList("bytes"), + Collections.singletonList( + logical(LogicalTypeFamily.BINARY_STRING)))) + .outputTypeStrategy(nullableIfArgs(explicit(DataTypes.GEOGRAPHY()))) + .runtimeClass( + "org.apache.flink.table.runtime.functions.scalar.StGeogFromWkbFunction") + .build(); + + public static final BuiltInFunctionDefinition ST_ASTEXT = + BuiltInFunctionDefinition.newBuilder() + .name("ST_ASTEXT") + .kind(SCALAR) + .inputTypeStrategy( + sequence( + Collections.singletonList("geography"), + Collections.singletonList(logical(LogicalTypeRoot.GEOGRAPHY)))) + .outputTypeStrategy(nullableIfArgs(explicit(DataTypes.STRING()))) + .runtimeClass( + "org.apache.flink.table.runtime.functions.scalar.StAsTextFunction") + .build(); + + public static final BuiltInFunctionDefinition ST_ASWKB = + BuiltInFunctionDefinition.newBuilder() + .name("ST_ASWKB") + .kind(SCALAR) + .inputTypeStrategy( + sequence( + Collections.singletonList("geography"), + Collections.singletonList(logical(LogicalTypeRoot.GEOGRAPHY)))) + .outputTypeStrategy(nullableIfArgs(explicit(DataTypes.BYTES()))) + .runtimeClass("org.apache.flink.table.runtime.functions.scalar.StAsWkbFunction") + .build(); + // -------------------------------------------------------------------------------------------- // Bitmap functions // -------------------------------------------------------------------------------------------- diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/GeographyType.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/GeographyType.java new file mode 100644 index 00000000000000..403ee88e0cfccd --- /dev/null +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/GeographyType.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.types.logical; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.table.data.GeographyData; + +import java.util.Collections; +import java.util.List; + +/** + * Data type of geography data. + * + *

The serializable string representation of this type is {@code GEOGRAPHY}. + */ +@PublicEvolving +public final class GeographyType extends LogicalType { + + private static final long serialVersionUID = 1L; + + private static final String FORMAT = "GEOGRAPHY"; + + private static final Class INPUT_OUTPUT_CONVERSION = GeographyData.class; + + public GeographyType(boolean isNullable) { + super(isNullable, LogicalTypeRoot.GEOGRAPHY); + } + + public GeographyType() { + this(true); + } + + @Override + public LogicalType copy(boolean isNullable) { + return new GeographyType(isNullable); + } + + @Override + public String asSerializableString() { + return withNullability(FORMAT); + } + + @Override + public boolean supportsInputConversion(Class clazz) { + return INPUT_OUTPUT_CONVERSION.isAssignableFrom(clazz); + } + + @Override + public boolean supportsOutputConversion(Class clazz) { + return INPUT_OUTPUT_CONVERSION.isAssignableFrom(clazz); + } + + @Override + public Class getDefaultConversion() { + return INPUT_OUTPUT_CONVERSION; + } + + @Override + public List getChildren() { + return Collections.emptyList(); + } + + @Override + public R accept(LogicalTypeVisitor visitor) { + return visitor.visit(this); + } +} diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/LogicalTypeRoot.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/LogicalTypeRoot.java index 6c823add433bfb..c21abce4af7461 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/LogicalTypeRoot.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/LogicalTypeRoot.java @@ -145,7 +145,9 @@ public enum LogicalTypeRoot { VARIANT(LogicalTypeFamily.EXTENSION), - BITMAP(LogicalTypeFamily.EXTENSION); + BITMAP(LogicalTypeFamily.EXTENSION), + + GEOGRAPHY(LogicalTypeFamily.EXTENSION); private final Set families; diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/LogicalTypeVisitor.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/LogicalTypeVisitor.java index 6a0e5614466d11..b35ffe05541f44 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/LogicalTypeVisitor.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/LogicalTypeVisitor.java @@ -101,5 +101,9 @@ default R visit(BitmapType bitmapType) { return visit((LogicalType) bitmapType); } + default R visit(GeographyType geographyType) { + return visit((LogicalType) geographyType); + } + R visit(LogicalType other); } diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeDefaultVisitor.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeDefaultVisitor.java index 056ec5953ccdc3..43237c443f56e9 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeDefaultVisitor.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeDefaultVisitor.java @@ -31,6 +31,7 @@ import org.apache.flink.table.types.logical.DistinctType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; @@ -203,6 +204,11 @@ public R visit(BitmapType bitmapType) { return defaultMethod(bitmapType); } + @Override + public R visit(GeographyType geographyType) { + return defaultMethod(geographyType); + } + @Override public R visit(LogicalType other) { return defaultMethod(other); diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeParser.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeParser.java index 3711ac76f4fd2a..dfa2e7cf358147 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeParser.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeParser.java @@ -36,6 +36,7 @@ import org.apache.flink.table.types.logical.DescriptorType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LegacyTypeInformationType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; @@ -336,7 +337,8 @@ private enum Keyword { DESCRIPTOR, STRUCTURED, VARIANT, - BITMAP + BITMAP, + GEOGRAPHY } private static final Set KEYWORDS = @@ -586,6 +588,8 @@ private LogicalType parseTypeByKeyword() { return new VariantType(); case BITMAP: return new BitmapType(); + case GEOGRAPHY: + return new GeographyType(); default: throw parsingError("Unsupported type: " + token().value); } diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeUtils.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeUtils.java index 98629b15fc1770..fa994ed160f3c7 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeUtils.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeUtils.java @@ -21,6 +21,7 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.table.data.ArrayData; import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.MapData; import org.apache.flink.table.data.RawValueData; import org.apache.flink.table.data.RowData; @@ -115,6 +116,8 @@ public static Class toInternalConversionClass(LogicalType type) { return Variant.class; case BITMAP: return Bitmap.class; + case GEOGRAPHY: + return GeographyData.class; case SYMBOL: case UNRESOLVED: default: diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/utils/LogicalTypeDataTypeConverter.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/utils/LogicalTypeDataTypeConverter.java index 9aa83f3cbee1d3..929aaec786f490 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/utils/LogicalTypeDataTypeConverter.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/utils/LogicalTypeDataTypeConverter.java @@ -38,6 +38,7 @@ import org.apache.flink.table.types.logical.DistinctType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; @@ -265,6 +266,11 @@ public DataType visit(BitmapType bitmapType) { return new AtomicDataType(bitmapType); } + @Override + public DataType visit(GeographyType geographyType) { + return new AtomicDataType(geographyType); + } + @Override public DataType visit(LogicalType other) { if (other.is(LogicalTypeRoot.UNRESOLVED)) { diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/api/SchemaTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/api/SchemaTest.java index 7b54e40f387935..7b4aef14cc3a80 100644 --- a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/api/SchemaTest.java +++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/api/SchemaTest.java @@ -51,5 +51,21 @@ void testFromResolvedSchema() { assertThat(newSchema.resolve(new TestSchemaResolver())).isEqualTo(originalSchema); } + + @Test + void testGeographyColumn() { + final Schema schema = + Schema.newBuilder() + .column("location", DataTypes.GEOGRAPHY()) + .column("required_location", DataTypes.GEOGRAPHY().notNull()) + .build(); + + assertThat(schema.resolve(new TestSchemaResolver())) + .isEqualTo( + ResolvedSchema.of( + Column.physical("location", DataTypes.GEOGRAPHY()), + Column.physical( + "required_location", DataTypes.GEOGRAPHY().notNull()))); + } } } diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/data/binary/BinaryGeographyDataTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/data/binary/BinaryGeographyDataTest.java new file mode 100644 index 00000000000000..4e45d2d28549aa --- /dev/null +++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/data/binary/BinaryGeographyDataTest.java @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.data.binary; + +import org.apache.flink.core.memory.MemorySegment; +import org.apache.flink.core.memory.MemorySegmentFactory; +import org.apache.flink.table.api.TableRuntimeException; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.GeographyData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.types.logical.GeographyType; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.Arrays; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link BinaryGeographyData}. */ +class BinaryGeographyDataTest { + + private static final int BIG_ENDIAN = 0; + private static final int LITTLE_ENDIAN = 1; + private static final byte[] POINT_WKB = pointWkb(LITTLE_ENDIAN); + + @Test + void testValidWkbRoundTrip() { + final GeographyData geography = GeographyData.fromBytes(POINT_WKB); + + assertThat(geography).isInstanceOf(BinaryGeographyData.class); + assertThat(geography.toBytes()).isEqualTo(POINT_WKB); + assertThat(geography.subtypeId()).isEqualTo(GeographyData.POINT); + assertThat(geography.sizeInBytes()).isEqualTo(POINT_WKB.length); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("supportedSubtypePayloads") + void testSupported2dSubtypePayloads(String name, int subtypeId, byte[] wkb) { + final BinaryGeographyData geography = BinaryGeographyData.fromBytes(wkb); + + assertThat(geography.subtypeId()).isEqualTo(subtypeId); + assertThat(geography.toBytes()).isEqualTo(wkb); + assertThat(geography.sizeInBytes()).isEqualTo(wkb.length); + } + + static Stream supportedSubtypePayloads() { + return Stream.of( + Arguments.of("Point", GeographyData.POINT, pointWkb(LITTLE_ENDIAN)), + Arguments.of("LineString", GeographyData.LINE_STRING, lineStringWkb(LITTLE_ENDIAN)), + Arguments.of("Polygon", GeographyData.POLYGON, polygonWkb(LITTLE_ENDIAN)), + Arguments.of("MultiPoint", GeographyData.MULTI_POINT, multiPointWkb(LITTLE_ENDIAN)), + Arguments.of( + "MultiLineString", + GeographyData.MULTI_LINE_STRING, + multiLineStringWkb(LITTLE_ENDIAN)), + Arguments.of( + "MultiPolygon", + GeographyData.MULTI_POLYGON, + multiPolygonWkb(LITTLE_ENDIAN)), + Arguments.of( + "GeometryCollection", + GeographyData.GEOMETRY_COLLECTION, + geometryCollectionWkb(LITTLE_ENDIAN))); + } + + @Test + void testBigEndianSubtypeExtraction() { + final BinaryGeographyData point = BinaryGeographyData.fromBytes(pointWkb(BIG_ENDIAN)); + final BinaryGeographyData lineString = + BinaryGeographyData.fromBytes(lineStringWkb(BIG_ENDIAN)); + + assertThat(point.subtypeId()).isEqualTo(GeographyData.POINT); + assertThat(lineString.subtypeId()).isEqualTo(GeographyData.LINE_STRING); + } + + @Test + void testNullHandling() { + assertThat(GeographyData.fromBytes((byte[]) null)).isNull(); + assertThat(GeographyData.fromBytes(null, 0, 0)).isNull(); + assertThat(BinaryGeographyData.fromBytes((byte[]) null)).isNull(); + assertThat(BinaryGeographyData.fromBytes(null, 0, 0)).isNull(); + } + + @Test + void testGenericRowDataNullPath() { + final RowData.FieldGetter getter = RowData.createFieldGetter(new GeographyType(), 0); + final GenericRowData nullRow = GenericRowData.of((Object) null); + final GeographyData geography = GeographyData.fromBytes(POINT_WKB); + final GenericRowData valueRow = GenericRowData.of(geography); + + assertThat(getter.getFieldOrNull(nullRow)).isNull(); + assertThat(getter.getFieldOrNull(valueRow)).isSameAs(geography); + } + + @Test + void testBinaryRowDataNullPath() { + final RowData.FieldGetter getter = RowData.createFieldGetter(new GeographyType(), 0); + final int fixedLength = BinaryRowData.calculateFixPartSizeInBytes(1); + final BinaryRowData nullRow = new BinaryRowData(1); + nullRow.pointTo(MemorySegmentFactory.wrap(new byte[fixedLength]), 0, fixedLength); + nullRow.setNullAt(0); + + assertThat(getter.getFieldOrNull(nullRow)).isNull(); + } + + @Test + void testBinaryRowDataValuePath() { + final BinaryRowData row = binaryRowWithGeography(POINT_WKB); + + assertThat(row.getGeography(0).toBytes()).isEqualTo(POINT_WKB); + assertThat(row.getGeography(0).subtypeId()).isEqualTo(GeographyData.POINT); + } + + @Test + void testByteRangeCopiesOnlySelectedPayload() { + final byte[] bytes = concat(bytes(42), POINT_WKB, bytes(99)); + final BinaryGeographyData geography = + BinaryGeographyData.fromBytes(bytes, 1, POINT_WKB.length); + + bytes[1] = 0; + + assertThat(geography.toBytes()).isEqualTo(POINT_WKB); + assertThat(geography.subtypeId()).isEqualTo(GeographyData.POINT); + } + + @Test + void testFromAddressSupportsSegmentBoundary() { + final byte[] first = concat(new byte[14], Arrays.copyOfRange(POINT_WKB, 0, 2)); + final byte[] second = Arrays.copyOfRange(POINT_WKB, 2, 18); + final byte[] third = + concat(Arrays.copyOfRange(POINT_WKB, 18, POINT_WKB.length), new byte[13]); + final MemorySegment[] segments = + new MemorySegment[] { + MemorySegmentFactory.wrap(first), + MemorySegmentFactory.wrap(second), + MemorySegmentFactory.wrap(third) + }; + + final BinaryGeographyData geography = + BinaryGeographyData.fromAddress(segments, 14, POINT_WKB.length); + + assertThat(geography.toBytes()).isEqualTo(POINT_WKB); + assertThat(geography.subtypeId()).isEqualTo(GeographyData.POINT); + } + + @Test + void testMalformedPayloadBoundaries() { + assertThatThrownBy(() -> BinaryGeographyData.fromBytes(bytes())) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Incomplete WKB header"); + + assertThatThrownBy(() -> BinaryGeographyData.fromBytes(bytes(1, 1, 0, 0))) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Incomplete WKB header"); + + assertThatThrownBy(() -> BinaryGeographyData.fromBytes(bytes(2, 1, 0, 0, 0))) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Unsupported byte order 2"); + + assertThatThrownBy(() -> BinaryGeographyData.fromBytes(header(LITTLE_ENDIAN, 0))) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Unsupported geography subtype ID 0"); + + assertThatThrownBy(() -> BinaryGeographyData.fromBytes(header(LITTLE_ENDIAN, 8))) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Unsupported geography subtype ID 8"); + } + + @Test + void testStructurallyIncompletePayloads() { + assertThatThrownBy( + () -> + BinaryGeographyData.fromBytes( + header(LITTLE_ENDIAN, GeographyData.POINT))) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Incomplete POINT coordinates"); + + assertThatThrownBy( + () -> + BinaryGeographyData.fromBytes( + concat( + header(LITTLE_ENDIAN, GeographyData.LINE_STRING), + unsignedInt(LITTLE_ENDIAN, 1), + new byte[15]))) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Incomplete coordinate sequence"); + + assertThatThrownBy( + () -> + BinaryGeographyData.fromBytes( + concat( + header(LITTLE_ENDIAN, GeographyData.MULTI_POINT), + unsignedInt(LITTLE_ENDIAN, 1), + lineStringWkb(LITTLE_ENDIAN)))) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Expected nested subtype ID 1 but found 2"); + + assertThatThrownBy( + () -> + BinaryGeographyData.fromBytes( + concat(pointWkb(LITTLE_ENDIAN), bytes(42)))) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("trailing byte"); + } + + @Test + void testInvalidByteRanges() { + assertThatThrownBy(() -> BinaryGeographyData.fromBytes(POINT_WKB, -1, POINT_WKB.length)) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Invalid ISO WKB byte range"); + + assertThatThrownBy(() -> BinaryGeographyData.fromBytes(POINT_WKB, 0, POINT_WKB.length + 1)) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Invalid ISO WKB byte range"); + } + + private static BinaryRowData binaryRowWithGeography(byte[] wkb) { + final int fixedLength = BinaryRowData.calculateFixPartSizeInBytes(1); + final byte[] rowBytes = new byte[fixedLength + wkb.length]; + final MemorySegment segment = MemorySegmentFactory.wrap(rowBytes); + segment.putLong(8, ((long) fixedLength << 32) | wkb.length); + segment.put(fixedLength, wkb, 0, wkb.length); + + final BinaryRowData row = new BinaryRowData(1); + row.pointTo(segment, 0, rowBytes.length); + return row; + } + + private static byte[] pointWkb(int byteOrder) { + return concat(header(byteOrder, GeographyData.POINT), new byte[16]); + } + + private static byte[] lineStringWkb(int byteOrder) { + return concat( + header(byteOrder, GeographyData.LINE_STRING), + unsignedInt(byteOrder, 2), + new byte[32]); + } + + private static byte[] polygonWkb(int byteOrder) { + return concat( + header(byteOrder, GeographyData.POLYGON), + unsignedInt(byteOrder, 1), + unsignedInt(byteOrder, 4), + new byte[64]); + } + + private static byte[] multiPointWkb(int byteOrder) { + return concat( + header(byteOrder, GeographyData.MULTI_POINT), + unsignedInt(byteOrder, 1), + pointWkb(byteOrder)); + } + + private static byte[] multiLineStringWkb(int byteOrder) { + return concat( + header(byteOrder, GeographyData.MULTI_LINE_STRING), + unsignedInt(byteOrder, 1), + lineStringWkb(byteOrder)); + } + + private static byte[] multiPolygonWkb(int byteOrder) { + return concat( + header(byteOrder, GeographyData.MULTI_POLYGON), + unsignedInt(byteOrder, 1), + polygonWkb(byteOrder)); + } + + private static byte[] geometryCollectionWkb(int byteOrder) { + return concat( + header(byteOrder, GeographyData.GEOMETRY_COLLECTION), + unsignedInt(byteOrder, 2), + pointWkb(byteOrder), + lineStringWkb(byteOrder)); + } + + private static byte[] header(int byteOrder, int subtypeId) { + return concat(bytes(byteOrder), unsignedInt(byteOrder, subtypeId)); + } + + private static byte[] unsignedInt(int byteOrder, int value) { + if (byteOrder == LITTLE_ENDIAN) { + return bytes(value, value >>> 8, value >>> 16, value >>> 24); + } + return bytes(value >>> 24, value >>> 16, value >>> 8, value); + } + + private static byte[] concat(byte[]... values) { + int length = 0; + for (byte[] value : values) { + length += value.length; + } + + final byte[] result = new byte[length]; + int offset = 0; + for (byte[] value : values) { + System.arraycopy(value, 0, result, offset, value.length); + offset += value.length; + } + return result; + } + + private static byte[] bytes(int... values) { + final byte[] result = new byte[values.length]; + for (int i = 0; i < values.length; i++) { + result[i] = (byte) values[i]; + } + return result; + } +} diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/DataTypesTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/DataTypesTest.java index 8c274ed719137e..3c0bf063631459 100644 --- a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/DataTypesTest.java +++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/DataTypesTest.java @@ -21,6 +21,7 @@ import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.api.common.typeutils.base.VoidSerializer; import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.types.logical.ArrayType; import org.apache.flink.table.types.logical.BigIntType; import org.apache.flink.table.types.logical.BinaryType; @@ -32,6 +33,7 @@ import org.apache.flink.table.types.logical.DecimalType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; @@ -82,6 +84,7 @@ import static org.apache.flink.table.api.DataTypes.DOUBLE; import static org.apache.flink.table.api.DataTypes.FIELD; import static org.apache.flink.table.api.DataTypes.FLOAT; +import static org.apache.flink.table.api.DataTypes.GEOGRAPHY; import static org.apache.flink.table.api.DataTypes.INT; import static org.apache.flink.table.api.DataTypes.INTERVAL; import static org.apache.flink.table.api.DataTypes.MAP; @@ -231,6 +234,9 @@ private static Stream testData() { TestSpec.forDataType(BITMAP()) .expectLogicalType(new BitmapType()) .expectConversionClass(Bitmap.class), + TestSpec.forDataType(GEOGRAPHY()) + .expectLogicalType(new GeographyType()) + .expectConversionClass(GeographyData.class), TestSpec.forUnresolvedDataType(RAW(Types.VOID)) .expectUnresolvedString("[RAW('java.lang.Void', '?')]") .lookupReturns(dummyRaw(Void.class)) diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeCastsTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeCastsTest.java index 6fb3574b1f59f6..89cc1341dfe2a8 100644 --- a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeCastsTest.java +++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeCastsTest.java @@ -31,6 +31,7 @@ import org.apache.flink.table.types.logical.DecimalType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; @@ -295,7 +296,13 @@ private static Stream testData() { new VariantType(), new MapType(new IntType(), new CharType()), false, - false)); + false), + + // GEOGRAPHY construction and serialization require explicit functions. + Arguments.of(new GeographyType(), VarCharType.STRING_TYPE, false, false), + Arguments.of(VarCharType.STRING_TYPE, new GeographyType(), false, false), + Arguments.of(new GeographyType(), new VarBinaryType(), false, false), + Arguments.of(new VarBinaryType(), new GeographyType(), false, false)); } @ParameterizedTest(name = "{index}: [From: {0}, To: {1}, Implicit: {2}, Explicit: {3}]") diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeParserTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeParserTest.java index 258b397c871878..e1c96b1fbc9e7c 100644 --- a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeParserTest.java +++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeParserTest.java @@ -39,6 +39,7 @@ import org.apache.flink.table.types.logical.DescriptorType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LegacyTypeInformationType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; @@ -309,6 +310,8 @@ private static Stream testData() { TestSpec.forString("VARIANT NOT NULL").expectType(new VariantType(false)), TestSpec.forString("BITMAP").expectType(new BitmapType()), TestSpec.forString("BITMAP NOT NULL").expectType(new BitmapType(false)), + TestSpec.forString("GEOGRAPHY").expectType(new GeographyType()), + TestSpec.forString("GEOGRAPHY NOT NULL").expectType(new GeographyType(false)), // error message testing diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypesTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypesTest.java index ee40463cf0b92d..b608d923cc3b6a 100644 --- a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypesTest.java +++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypesTest.java @@ -26,6 +26,8 @@ import org.apache.flink.table.api.ValidationException; import org.apache.flink.table.catalog.ObjectIdentifier; import org.apache.flink.table.catalog.UnresolvedIdentifier; +import org.apache.flink.table.data.GeographyData; +import org.apache.flink.table.data.binary.BinaryGeographyData; import org.apache.flink.table.expressions.TimeIntervalUnit; import org.apache.flink.table.legacy.types.logical.TypeInformationRawType; import org.apache.flink.table.types.logical.ArrayType; @@ -40,6 +42,7 @@ import org.apache.flink.table.types.logical.DistinctType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; @@ -629,6 +632,20 @@ void testBitmapType() { new BitmapType(false))); } + @Test + void testGeographyType() { + assertThat(new GeographyType()) + .isJavaSerializable() + .satisfies( + baseAssertions( + "GEOGRAPHY", + "GEOGRAPHY", + new Class[] {GeographyData.class, BinaryGeographyData.class}, + new Class[] {GeographyData.class, BinaryGeographyData.class}, + new LogicalType[] {}, + new GeographyType(false))); + } + @Test void testTypeInformationRawType() { final TypeInformationRawType rawType = diff --git a/flink-table/flink-table-planner/pom.xml b/flink-table/flink-table-planner/pom.xml index 450dc204ec69ed..584b5aa31f22e8 100644 --- a/flink-table/flink-table-planner/pom.xml +++ b/flink-table/flink-table-planner/pom.xml @@ -189,6 +189,19 @@ under the License. test + + org.apache.flink + flink-table-code-splitter + ${project.version} + test + + + org.antlr + antlr4-runtime + 4.7 + test + + org.apache.flink @@ -235,6 +248,12 @@ under the License. test-jar test + + org.apache.flink + flink-shaded-jsonpath + ${flink.shaded.jsonpath.version}-${flink.shaded.version} + test + org.apache.flink diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/fun/SqlCastFunction.java b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/fun/SqlCastFunction.java index 7b6112f9672dc8..d819a145e94cdd 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/fun/SqlCastFunction.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/fun/SqlCastFunction.java @@ -269,16 +269,22 @@ public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFail private boolean canCastFrom(RelDataType toType, RelDataType fromType) { SqlTypeName fromTypeName = fromType.getSqlTypeName(); + SqlTypeName toTypeName = toType.getSqlTypeName(); // Cast to Variant is not support at the moment. // TODO: Support cast to variant (FLINK-37925,FLINK-37926) - if (toType.getSqlTypeName() == SqlTypeName.VARIANT) { + if (toTypeName == SqlTypeName.VARIANT) { return false; } // Cast to BITMAP is not supported at the moment. if (toType instanceof BitmapRelDataType) { return false; } + if (toTypeName == SqlTypeName.OTHER) { + return LogicalTypeCasts.supportsExplicitCast( + FlinkTypeFactory.toLogicalType(fromType), + FlinkTypeFactory.toLogicalType(toType)); + } switch (fromTypeName) { case ARRAY: case MAP: diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/FlinkTypeFactory.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/FlinkTypeFactory.java index f5bd64adeefe22..c101181e2e65b7 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/FlinkTypeFactory.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/FlinkTypeFactory.java @@ -29,6 +29,7 @@ import org.apache.flink.table.legacy.types.logical.TypeInformationRawType; import org.apache.flink.table.planner.plan.schema.BitmapRelDataType; import org.apache.flink.table.planner.plan.schema.GenericRelDataType; +import org.apache.flink.table.planner.plan.schema.GeographyRelDataType; import org.apache.flink.table.planner.plan.schema.RawRelDataType; import org.apache.flink.table.planner.plan.schema.StructuredRelDataType; import org.apache.flink.table.planner.plan.schema.TimeIndicatorRelDataType; @@ -47,6 +48,7 @@ import org.apache.flink.table.types.logical.DescriptorType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LegacyTypeInformationType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; @@ -66,6 +68,7 @@ import org.apache.flink.table.types.logical.VarBinaryType; import org.apache.flink.table.types.logical.VarCharType; import org.apache.flink.table.types.logical.VariantType; +import org.apache.flink.table.types.logical.utils.LogicalTypeMerging; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.table.utils.TableSchemaUtils; import org.apache.flink.util.Preconditions; @@ -161,6 +164,11 @@ public RelDataType createBitmapType() { return canonize(new BitmapRelDataType(new BitmapType())); } + @Override + public RelDataType createGeographyType() { + return canonize(new GeographyRelDataType(new GeographyType())); + } + @Override public RelDataType createArrayType(RelDataType elementType, long maxCardinality) { // Just validate type, make sure there is a failure in validate phase. @@ -231,6 +239,8 @@ public RelDataType createTypeWithNullability(RelDataType relDataType, boolean is newType = ((StructuredRelDataType) relDataType).createWithNullability(isNullable); } else if (relDataType instanceof BitmapRelDataType) { newType = ((BitmapRelDataType) relDataType).createWithNullability(isNullable); + } else if (relDataType instanceof GeographyRelDataType) { + newType = ((GeographyRelDataType) relDataType).createWithNullability(isNullable); } else if (relDataType instanceof GenericRelDataType) { final GenericRelDataType generic = (GenericRelDataType) relDataType; newType = new GenericRelDataType(generic.genericType(), isNullable, getTypeSystem()); @@ -255,8 +265,20 @@ public RelDataType createTypeWithNullability(RelDataType relDataType, boolean is @Override public RelDataType leastRestrictive(List types) { final Optional resolved = resolveAllIdenticalTypes(types); - final RelDataType leastRestrictive = - resolved.orElseGet(() -> super.leastRestrictive(types)); + if (resolved.isPresent()) { + return normalizeLeastRestrictive(resolved.get()); + } + + if (containsFlinkExtensionType(types)) { + return normalizeLeastRestrictive( + resolveCommonTypeForFlinkExtensions(types).orElse(null)); + } + + final RelDataType leastRestrictive = super.leastRestrictive(types); + return normalizeLeastRestrictive(leastRestrictive); + } + + private RelDataType normalizeLeastRestrictive(RelDataType leastRestrictive) { // NULL is reserved for untyped literals only if (leastRestrictive == null || leastRestrictive.getSqlTypeName() == SqlTypeName.NULL) { return null; @@ -264,6 +286,24 @@ public RelDataType leastRestrictive(List types) { return leastRestrictive; } + private Optional resolveCommonTypeForFlinkExtensions(List types) { + return LogicalTypeMerging.findCommonType( + types.stream() + .map(FlinkTypeFactory::toLogicalType) + .collect(Collectors.toList())) + .map(this::createFieldTypeFromLogicalType); + } + + private boolean containsFlinkExtensionType(List types) { + return types.stream().anyMatch(FlinkTypeFactory::isFlinkExtensionType); + } + + private static boolean isFlinkExtensionType(RelDataType type) { + return type instanceof RawRelDataType + || type instanceof BitmapRelDataType + || type instanceof GeographyRelDataType; + } + private Optional resolveAllIdenticalTypes(List types) { final RelDataType head = types.get(0); // check if all types are the same @@ -493,6 +533,9 @@ private RelDataType newRelDataType(LogicalType logicalType) { case BITMAP: return new BitmapRelDataType((BitmapType) logicalType); + case GEOGRAPHY: + return new GeographyRelDataType((GeographyType) logicalType); + default: throw new TableException("Type is not supported: " + logicalType); } @@ -866,6 +909,8 @@ private static LogicalType toLogicalTypeWithoutNullability(RelDataType relDataTy return ((RawRelDataType) relDataType).getRawType(); } else if (relDataType instanceof BitmapRelDataType) { return ((BitmapRelDataType) relDataType).getBitmapType(); + } else if (relDataType instanceof GeographyRelDataType) { + return ((GeographyRelDataType) relDataType).getGeographyType(); } else { throw new TableException("Type is not supported: " + relDataType); } diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/expressions/converter/ExpressionConverter.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/expressions/converter/ExpressionConverter.java index 3a532d6f15f5df..9e9b0da7a20e16 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/expressions/converter/ExpressionConverter.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/expressions/converter/ExpressionConverter.java @@ -18,11 +18,13 @@ package org.apache.flink.table.planner.expressions.converter; +import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.api.TableException; import org.apache.flink.table.catalog.Catalog; import org.apache.flink.table.catalog.ContextResolvedModel; import org.apache.flink.table.catalog.DataTypeFactory; import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.expressions.CallExpression; import org.apache.flink.table.expressions.Expression; import org.apache.flink.table.expressions.ExpressionVisitor; @@ -36,6 +38,7 @@ import org.apache.flink.table.expressions.ValueLiteralExpression; import org.apache.flink.table.factories.FactoryUtil; import org.apache.flink.table.factories.ModelProviderFactory; +import org.apache.flink.table.functions.BuiltInFunctionDefinitions; import org.apache.flink.table.ml.ModelProvider; import org.apache.flink.table.module.Module; import org.apache.flink.table.planner.calcite.FlinkContext; @@ -141,6 +144,10 @@ public RexNode visit(ValueLiteralExpression valueLiteral) { .collect(Collectors.toList())); } + if (type.getTypeRoot() == LogicalTypeRoot.GEOGRAPHY) { + return convertGeographyLiteral(valueLiteral); + } + Object value; switch (type.getTypeRoot()) { case DECIMAL: @@ -220,6 +227,28 @@ public RexNode visit(ValueLiteralExpression valueLiteral) { true); } + private RexNode convertGeographyLiteral(ValueLiteralExpression valueLiteral) { + final GeographyData geography = + valueLiteral + .getValueAs(GeographyData.class) + .orElseThrow( + () -> + new TableException( + String.format( + "GEOGRAPHY literals require values of class '%s' but found '%s'.", + GeographyData.class.getName(), + extractValue(valueLiteral, Object.class) + .getClass() + .getName()))); + return visit( + CallExpression.permanent( + BuiltInFunctionDefinitions.ST_GEOGFROMWKB, + List.of( + new ValueLiteralExpression( + geography.toBytes(), DataTypes.BYTES().notNull())), + valueLiteral.getOutputDataType())); + } + @Override public RexNode visit(FieldReferenceExpression fieldReference) { // We can not use inputCount+inputIndex+FieldIndex to construct field of calcite. diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/sql/FlinkSqlOperatorTable.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/sql/FlinkSqlOperatorTable.java index 68c950b867acbf..29a798bac816a5 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/sql/FlinkSqlOperatorTable.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/sql/FlinkSqlOperatorTable.java @@ -19,12 +19,14 @@ package org.apache.flink.table.planner.functions.sql; import org.apache.flink.table.api.TableException; +import org.apache.flink.table.functions.BuiltInFunctionDefinitions; import org.apache.flink.table.planner.calcite.FlinkTypeFactory; import org.apache.flink.table.planner.functions.sql.internal.SqlAuxiliaryGroupAggFunction; import org.apache.flink.table.planner.functions.sql.ml.SqlMLEvaluateTableFunction; import org.apache.flink.table.planner.functions.sql.ml.SqlVectorSearchTableFunction; import org.apache.flink.table.planner.plan.type.FlinkReturnTypes; import org.apache.flink.table.planner.plan.type.NumericExceptFirstOperandChecker; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.sql.SqlFunction; @@ -72,6 +74,16 @@ public class FlinkSqlOperatorTable extends ReflectiveSqlOperatorTable { /** The table of contains Flink-specific operators. */ private static final Map cachedInstances = new HashMap<>(); + private static final SqlReturnTypeInference GEOGRAPHY_NULLABLE_IF_ARGS = + opBinding -> { + boolean nullable = false; + for (int i = 0; i < opBinding.getOperandCount(); i++) { + nullable |= opBinding.getOperandType(i).isNullable(); + } + return ((FlinkTypeFactory) opBinding.getTypeFactory()) + .createFieldTypeFromLogicalType(new GeographyType(nullable)); + }; + /** Returns the Flink operator table, creating it if necessary. */ public static synchronized FlinkSqlOperatorTable instance(boolean isBatchMode) { FlinkSqlOperatorTable instance = cachedInstances.get(isBatchMode); @@ -867,6 +879,42 @@ public SqlSyntax getSyntax() { OperandTypes.family(SqlTypeFamily.BINARY, SqlTypeFamily.CHARACTER), SqlFunctionCategory.STRING); + // GEOGRAPHY FUNCTIONS + public static final SqlFunction ST_GEOGFROMTEXT = + BuiltInSqlFunction.newBuilder() + .name(BuiltInFunctionDefinitions.ST_GEOGFROMTEXT.getName()) + .returnType(GEOGRAPHY_NULLABLE_IF_ARGS) + .operandTypeChecker(OperandTypes.family(SqlTypeFamily.ANY)) + .category(SqlFunctionCategory.STRING) + .build(); + + public static final SqlFunction ST_GEOGFROMWKB = + BuiltInSqlFunction.newBuilder() + .name(BuiltInFunctionDefinitions.ST_GEOGFROMWKB.getName()) + .returnType(GEOGRAPHY_NULLABLE_IF_ARGS) + .operandTypeChecker(OperandTypes.family(SqlTypeFamily.ANY)) + .category(SqlFunctionCategory.STRING) + .build(); + + public static final SqlFunction ST_ASTEXT = + BuiltInSqlFunction.newBuilder() + .name(BuiltInFunctionDefinitions.ST_ASTEXT.getName()) + .returnType(VARCHAR_FORCE_NULLABLE) + .operandTypeChecker(OperandTypes.family(SqlTypeFamily.ANY)) + .category(SqlFunctionCategory.STRING) + .build(); + + public static final SqlFunction ST_ASWKB = + BuiltInSqlFunction.newBuilder() + .name(BuiltInFunctionDefinitions.ST_ASWKB.getName()) + .returnType( + ReturnTypes.cascade( + ReturnTypes.explicit(SqlTypeName.VARBINARY), + SqlTypeTransforms.TO_NULLABLE)) + .operandTypeChecker(OperandTypes.family(SqlTypeFamily.ANY)) + .category(SqlFunctionCategory.STRING) + .build(); + public static final SqlFunction INSTR = new SqlFunction( "INSTR", diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/utils/SqlValidatorUtils.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/utils/SqlValidatorUtils.java index 8fd5d8fee056c2..e1614d6e98e23f 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/utils/SqlValidatorUtils.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/utils/SqlValidatorUtils.java @@ -19,6 +19,7 @@ package org.apache.flink.table.planner.functions.utils; import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.planner.plan.schema.GeographyRelDataType; import org.apache.flink.types.Either; import org.apache.calcite.rel.type.RelDataType; @@ -158,7 +159,8 @@ private static void adjustTypeForMultisetConstructor( } else { elementType = oddType; } - if (operandTypes.get(i).equalsSansFieldNames(elementType)) { + if (operandTypes.get(i).equalsSansFieldNames(elementType) + || elementType instanceof GeographyRelDataType) { continue; } call.setOperand(i, castTo(operands.get(i), elementType)); diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/serde/LogicalTypeJsonSerializer.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/serde/LogicalTypeJsonSerializer.java index f06c38ba1b8b69..39f28a6f8c89f3 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/serde/LogicalTypeJsonSerializer.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/serde/LogicalTypeJsonSerializer.java @@ -546,6 +546,7 @@ protected Boolean defaultMethod(LogicalType logicalType) { case DESCRIPTOR: case BITMAP: case VARIANT: + case GEOGRAPHY: return true; default: // fall back to generic serialization diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/schema/GeographyRelDataType.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/schema/GeographyRelDataType.java new file mode 100644 index 00000000000000..41b586f5863240 --- /dev/null +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/schema/GeographyRelDataType.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.schema; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.types.logical.GeographyType; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.sql.type.AbstractSqlType; +import org.apache.calcite.sql.type.SqlTypeName; + +/** The {@link RelDataType} representation of a {@link GeographyType}. */ +@Internal +public final class GeographyRelDataType extends AbstractSqlType { + + private final GeographyType geographyType; + + public GeographyRelDataType(GeographyType geographyType) { + super(SqlTypeName.OTHER, geographyType.isNullable(), null); + this.geographyType = geographyType; + computeDigest(); + } + + public GeographyType getGeographyType() { + return geographyType; + } + + public GeographyRelDataType createWithNullability(boolean nullable) { + if (nullable == isNullable()) { + return this; + } + return new GeographyRelDataType((GeographyType) geographyType.copy(nullable)); + } + + @Override + protected void generateTypeString(StringBuilder sb, boolean withDetail) { + sb.append(geographyType.asSummaryString()); + } + + @Override + protected void computeDigest() { + final StringBuilder sb = new StringBuilder(); + generateTypeString(sb, true); + digest = sb.toString(); + } +} diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/typeutils/LogicalRelDataTypeConverter.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/typeutils/LogicalRelDataTypeConverter.java index cf78ce82068e53..98e4593975119f 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/typeutils/LogicalRelDataTypeConverter.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/typeutils/LogicalRelDataTypeConverter.java @@ -23,6 +23,7 @@ import org.apache.flink.table.catalog.DataTypeFactory; import org.apache.flink.table.planner.calcite.FlinkTypeFactory; import org.apache.flink.table.planner.plan.schema.BitmapRelDataType; +import org.apache.flink.table.planner.plan.schema.GeographyRelDataType; import org.apache.flink.table.planner.plan.schema.RawRelDataType; import org.apache.flink.table.planner.plan.schema.StructuredRelDataType; import org.apache.flink.table.planner.plan.schema.TimeIndicatorRelDataType; @@ -40,6 +41,7 @@ import org.apache.flink.table.types.logical.DistinctType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; @@ -472,6 +474,11 @@ public RelDataType visit(BitmapType bitmapType) { return new BitmapRelDataType(bitmapType); } + @Override + public RelDataType visit(GeographyType geographyType) { + return new GeographyRelDataType(geographyType); + } + @Override public RelDataType visit(LogicalType other) { throw new TableException( @@ -604,6 +611,8 @@ private static LogicalType toLogicalTypeNotNull( return ((RawRelDataType) relDataType).getRawType(); } else if (relDataType instanceof BitmapRelDataType) { return ((BitmapRelDataType) relDataType).getBitmapType(); + } else if (relDataType instanceof GeographyRelDataType) { + return ((GeographyRelDataType) relDataType).getGeographyType(); } // fall through case REAL: diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/CodeGenUtils.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/CodeGenUtils.scala index aacb8db5a0d4a6..a1f5144905803f 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/CodeGenUtils.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/CodeGenUtils.scala @@ -283,6 +283,7 @@ object CodeGenUtils { case DESCRIPTOR => className[ColumnList] case VARIANT => className[Variant] case BITMAP => className[Bitmap] + case GEOGRAPHY => className[GeographyData] case SYMBOL | UNRESOLVED => throw new IllegalArgumentException("Illegal type: " + t) } @@ -390,6 +391,8 @@ object CodeGenUtils { s"$term.toObject($serTerm).hashCode()" case BITMAP => s"$term.hashCode()" + case GEOGRAPHY => + s"$term.hashCode()" case NULL | SYMBOL | UNRESOLVED => throw new IllegalArgumentException("Illegal type: " + t) } @@ -542,6 +545,8 @@ object CodeGenUtils { s"$rowTerm.getVariant($indexTerm)" case BITMAP => s"$rowTerm.getBitmap($indexTerm)" + case GEOGRAPHY => + s"$rowTerm.getGeography($indexTerm)" case NULL | SYMBOL | UNRESOLVED => throw new IllegalArgumentException("Illegal type: " + t) } @@ -839,6 +844,8 @@ object CodeGenUtils { s"$writerTerm.writeVariant($indexTerm, $fieldValTerm)" case BITMAP => s"$writerTerm.writeBitmap($indexTerm, $fieldValTerm)" + case GEOGRAPHY => + s"$writerTerm.writeGeography($indexTerm, $fieldValTerm)" case NULL | SYMBOL | UNRESOLVED => throw new IllegalArgumentException("Illegal type: " + t); } diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/CompiledPlanITCase.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/CompiledPlanITCase.java index 0d5c13d48a538b..aa0039b7bd16bf 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/CompiledPlanITCase.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/CompiledPlanITCase.java @@ -86,6 +86,33 @@ void testCompilePlanSql() throws IOException { .isEqualTo(getPreparedToCompareCompiledPlan(expected)); } + @Test + void testCompilePlanSqlWithGeographyColumns() { + tableEnv.executeSql( + "CREATE TABLE GeoSource (" + + "id INT, " + + "location GEOGRAPHY, " + + "required_location GEOGRAPHY NOT NULL" + + ") WITH (" + + "'connector' = 'values', " + + "'bounded' = 'false')"); + tableEnv.executeSql( + "CREATE TABLE GeoSink (" + + "id INT, " + + "location GEOGRAPHY, " + + "required_location GEOGRAPHY NOT NULL" + + ") WITH (" + + "'connector' = 'values', " + + "'table-sink-class' = 'DEFAULT')"); + + final CompiledPlan compiledPlan = + tableEnv.compilePlanSql("INSERT INTO GeoSink SELECT * FROM GeoSource"); + final String planJson = compiledPlan.asJsonString(); + + assertThat(planJson).contains("\"GEOGRAPHY\"", "\"GEOGRAPHY NOT NULL\""); + assertThat(tableEnv.loadPlan(PlanReference.fromJsonString(planJson))).isNotNull(); + } + @Test void testSourceTableWithHints() { CompiledPlan compiledPlan = diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/calcite/FlinkCalciteSqlValidatorTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/calcite/FlinkCalciteSqlValidatorTest.java index 831b3b6ecc480a..ed2fef1ce876cd 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/calcite/FlinkCalciteSqlValidatorTest.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/calcite/FlinkCalciteSqlValidatorTest.java @@ -26,7 +26,10 @@ import org.apache.flink.table.api.ValidationException; import org.apache.flink.table.functions.ScalarFunction; import org.apache.flink.table.planner.utils.PlannerMocks; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.calcite.rel.RelRoot; +import org.apache.calcite.sql.SqlNode; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -158,6 +161,14 @@ void testMixedPositionalAndNamedArguments() { .hasMessageContaining("Cannot mix positional and named arguments"); } + private LogicalType projectedLogicalType(String expression) { + SqlNode parsed = plannerMocks.getPlanner().parser().parse("SELECT " + expression + " AS c"); + SqlNode validated = plannerMocks.getPlanner().validate(parsed); + RelRoot relRoot = plannerMocks.getPlanner().rel(validated); + return FlinkTypeFactory.toLogicalType( + relRoot.rel.getRowType().getFieldList().get(0).getType()); + } + /** Scalar function with named arguments for the mixed-argument validation test. */ public static class NamedArgsScalarFunction extends ScalarFunction { @FunctionHint( diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/calcite/FlinkTypeFactoryTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/calcite/FlinkTypeFactoryTest.java index 6e0e7a8c3d3e23..a18a8d31baf024 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/calcite/FlinkTypeFactoryTest.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/calcite/FlinkTypeFactoryTest.java @@ -27,6 +27,7 @@ import org.apache.flink.table.legacy.types.logical.TypeInformationRawType; import org.apache.flink.table.types.logical.ArrayType; import org.apache.flink.table.types.logical.BigIntType; +import org.apache.flink.table.types.logical.BitmapType; import org.apache.flink.table.types.logical.BooleanType; import org.apache.flink.table.types.logical.CharType; import org.apache.flink.table.types.logical.DateType; @@ -34,6 +35,7 @@ import org.apache.flink.table.types.logical.DecimalType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; @@ -269,6 +271,37 @@ void testLeastRestrictive(List input, LogicalType expected) { .isEqualTo(typeFactory.createFieldTypeFromLogicalType(expected)); } + @Test + void testLeastRestrictiveGeographyNullability() { + FlinkTypeFactory typeFactory = + new FlinkTypeFactory( + Thread.currentThread().getContextClassLoader(), FlinkTypeSystem.INSTANCE); + + assertThat( + typeFactory.leastRestrictive( + Stream.of( + new GeographyType(false), + new GeographyType(true), + new NullType()) + .map(typeFactory::createFieldTypeFromLogicalType) + .collect(Collectors.toList()))) + .isEqualTo(typeFactory.createFieldTypeFromLogicalType(new GeographyType(true))); + } + + @Test + void testLeastRestrictiveIncompatibleExtensionTypes() { + FlinkTypeFactory typeFactory = + new FlinkTypeFactory( + Thread.currentThread().getContextClassLoader(), FlinkTypeSystem.INSTANCE); + + assertThat( + typeFactory.leastRestrictive( + Stream.of(new GeographyType(), new BitmapType()) + .map(typeFactory::createFieldTypeFromLogicalType) + .collect(Collectors.toList()))) + .isNull(); + } + public static class TestClass { public int f0; public String f1; diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/codegen/ProjectionCodeGeneratorGeographyTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/codegen/ProjectionCodeGeneratorGeographyTest.java new file mode 100644 index 00000000000000..652d0af25d6427 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/codegen/ProjectionCodeGeneratorGeographyTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.codegen; + +import org.apache.flink.configuration.Configuration; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.GeographyData; +import org.apache.flink.table.data.binary.BinaryRowData; +import org.apache.flink.table.runtime.generated.Projection; +import org.apache.flink.table.types.logical.GeographyType; +import org.apache.flink.table.types.logical.IntType; +import org.apache.flink.table.types.logical.RowType; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for generated projections involving {@link GeographyData}. */ +class ProjectionCodeGeneratorGeographyTest { + + private static final byte[] POINT_WKB = + new byte[] { + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, (byte) 0xF0, 0x3F, 0, 0, 0, 0, 0, 0, 0, 0x40 + }; + + @Test + void testGeneratedProjectionForGeography() { + RowType inputType = RowType.of(new IntType(), new GeographyType(), new GeographyType()); + RowType outputType = RowType.of(new GeographyType(), new GeographyType(), new IntType()); + GenericRowData input = GenericRowData.of(7, GeographyData.fromBytes(POINT_WKB), null); + + Projection projection = + ProjectionCodeGenerator.generateProjection( + new CodeGeneratorContext( + new Configuration(), + Thread.currentThread().getContextClassLoader()), + "GeographyProjection", + inputType, + outputType, + new int[] {1, 2, 0}) + .newInstance(Thread.currentThread().getContextClassLoader()); + + BinaryRowData output = (BinaryRowData) projection.apply(input); + + assertThat(output.getGeography(0).toBytes()).isEqualTo(POINT_WKB); + assertThat(output.isNullAt(1)).isTrue(); + assertThat(output.getInt(2)).isEqualTo(7); + } +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/expressions/LiteralExpressionsSerializationITCase.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/expressions/LiteralExpressionsSerializationITCase.java index 7737003f65f1f6..b8d05d7392baec 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/expressions/LiteralExpressionsSerializationITCase.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/expressions/LiteralExpressionsSerializationITCase.java @@ -23,6 +23,7 @@ import org.apache.flink.table.api.Table; import org.apache.flink.table.api.TableEnvironment; import org.apache.flink.table.api.TableResult; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.expressions.DefaultSqlFactory; import org.apache.flink.table.expressions.ResolvedExpression; import org.apache.flink.table.expressions.SqlFactory; @@ -42,17 +43,93 @@ import java.time.LocalTime; import java.time.Period; import java.time.temporal.ChronoUnit; +import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; +import static org.apache.flink.table.api.Expressions.array; import static org.apache.flink.table.api.Expressions.lit; +import static org.apache.flink.table.api.Expressions.map; import static org.apache.flink.table.api.Expressions.nullOf; +import static org.apache.flink.table.api.Expressions.row; import static org.assertj.core.api.Assertions.assertThat; /** Tests for {@link ResolvedExpression#asSerializableString(SqlFactory)}. */ @ExtendWith(MiniClusterExtension.class) public class LiteralExpressionsSerializationITCase { + private static final byte[] POINT_WKB = + new byte[] { + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, (byte) 0xF0, 0x3F, 0, 0, 0, 0, 0, 0, 0, 0x40 + }; + + private static final String POINT_WKB_HEX = "0101000000000000000000f03f0000000000000040"; + + @Test + void testGeographySqlSerialization() { + final TableEnvironment env = TableEnvironment.create(EnvironmentSettings.inStreamingMode()); + final GeographyData geography = GeographyData.fromBytes(POINT_WKB); + final Table t = + env.fromValues(1) + .select( + lit(geography, DataTypes.GEOGRAPHY().notNull()), + lit(null, DataTypes.GEOGRAPHY()), + array( + lit(geography, DataTypes.GEOGRAPHY().notNull()), + lit(null, DataTypes.GEOGRAPHY())), + map( + lit("a", DataTypes.STRING().notNull()), + lit(geography, DataTypes.GEOGRAPHY().notNull())), + row( + lit(geography, DataTypes.GEOGRAPHY().notNull()), + lit(null, DataTypes.GEOGRAPHY()))); + + final ProjectQueryOperation operation = (ProjectQueryOperation) t.getQueryOperation(); + final List expressions = + operation.getProjectList().stream() + .map( + resolvedExpression -> + resolvedExpression.asSerializableString( + DefaultSqlFactory.INSTANCE)) + .collect(Collectors.toList()); + + assertThat(expressions.get(0)).isEqualTo("ST_GEOGFROMWKB(X'" + POINT_WKB_HEX + "')"); + assertThat(expressions.get(1)).isEqualTo("CAST(NULL AS GEOGRAPHY)"); + assertThat(expressions.get(2)) + .contains("ST_GEOGFROMWKB(X'" + POINT_WKB_HEX + "')") + .contains("CAST(NULL AS GEOGRAPHY)"); + assertThat(expressions.get(3)) + .isEqualTo("MAP['a', ST_GEOGFROMWKB(X'" + POINT_WKB_HEX + "')]"); + assertThat(expressions.get(4)) + .contains("ST_GEOGFROMWKB(X'" + POINT_WKB_HEX + "')") + .contains("CAST(NULL AS GEOGRAPHY)"); + + final TableResult tableResult = + env.sqlQuery(String.format("SELECT %s", String.join(", ", expressions))).execute(); + final Row result = CollectionUtil.iteratorToList(tableResult.collect()).get(0); + + assertGeography(result.getField(0)); + assertThat(result.getField(1)).isNull(); + + final Object arrayField = result.getField(2); + final List arrayValue = + arrayField instanceof List + ? (List) arrayField + : Arrays.asList((Object[]) arrayField); + assertThat(arrayValue).hasSize(2); + assertGeography(arrayValue.get(0)); + assertThat(arrayValue.get(1)).isNull(); + + final Map mapValue = (Map) result.getField(3); + assertThat(mapValue).hasSize(1); + assertGeography(mapValue.get("a")); + + final Row rowValue = (Row) result.getField(4); + assertGeography(rowValue.getField(0)); + assertThat(rowValue.getField(1)).isNull(); + } + @Test void testSqlSerialization() { final TableEnvironment env = TableEnvironment.create(EnvironmentSettings.inStreamingMode()); @@ -192,4 +269,9 @@ void testSqlSerialization() { defaultPeriod, derivedLargePeriod)); } + + private static void assertGeography(Object value) { + assertThat(value).isInstanceOf(GeographyData.class); + assertThat(((GeographyData) value).toBytes()).isEqualTo(POINT_WKB); + } } diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/expressions/converter/ExpressionConverterTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/expressions/converter/ExpressionConverterTest.java index ffea6f95d5e2d3..588db6b5bc3d59 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/expressions/converter/ExpressionConverterTest.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/expressions/converter/ExpressionConverterTest.java @@ -19,11 +19,14 @@ package org.apache.flink.table.planner.expressions.converter; import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.expressions.TimePointUnit; import org.apache.flink.table.planner.delegation.PlannerContext; import org.apache.flink.table.planner.utils.PlannerMocks; import org.apache.calcite.avatica.util.TimeUnit; +import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.type.SqlTypeName; @@ -42,10 +45,16 @@ import static org.apache.flink.table.expressions.ApiExpressionUtils.valueLiteral; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; /** Test for {@link ExpressionConverter}. */ class ExpressionConverterTest { + private static final byte[] POINT_WKB = + new byte[] { + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, (byte) 0xF0, 0x3F, 0, 0, 0, 0, 0, 0, 0, 0x40 + }; + private final PlannerContext plannerContext = PlannerMocks.create().getPlannerContext(); private final ExpressionConverter converter = new ExpressionConverter(plannerContext.createRelBuilder()); @@ -194,4 +203,39 @@ void testSymbolLiteral() { assertThat(((RexLiteral) rex).getValueAs(TimeUnit.class)).isEqualTo(TimeUnit.MICROSECOND); assertThat(rex.getType().getSqlTypeName()).isEqualTo(SqlTypeName.SYMBOL); } + + @Test + void testGeographyLiteralUsesConstructorCall() { + RexNode rex = + converter.visit( + valueLiteral( + GeographyData.fromBytes(POINT_WKB), + DataTypes.GEOGRAPHY().notNull())); + + assertThat(rex).isInstanceOf(RexCall.class); + assertThat(rex.getType().getSqlTypeName()).isEqualTo(SqlTypeName.OTHER); + assertThat(rex.getType().isNullable()).isFalse(); + + RexCall call = (RexCall) rex; + assertThat(call.getOperator().getName()).isEqualTo("ST_GEOGFROMWKB"); + assertThat(((RexLiteral) call.getOperands().get(0)).getValueAs(byte[].class)) + .isEqualTo(POINT_WKB); + } + + @Test + void testNullableGeographyLiteral() { + RexNode rex = converter.visit(valueLiteral(null, DataTypes.GEOGRAPHY())); + + assertThat(rex).isInstanceOf(RexLiteral.class); + assertThat(rex.getType().getSqlTypeName()).isEqualTo(SqlTypeName.OTHER); + assertThat(rex.getType().isNullable()).isTrue(); + assertThat(((RexLiteral) rex).getValue()).isNull(); + } + + @Test + void testInvalidGeographyLiteralFailsValidation() { + assertThatThrownBy(() -> valueLiteral("POINT (1 2)", DataTypes.GEOGRAPHY().notNull())) + .isInstanceOf(ValidationException.class) + .hasMessageContaining("does not support a value literal"); + } } diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/CastFunctionMiscITCase.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/CastFunctionMiscITCase.java index 218d911e881dba..f37a8acfa4ac47 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/CastFunctionMiscITCase.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/CastFunctionMiscITCase.java @@ -39,6 +39,7 @@ import static org.apache.flink.table.api.DataTypes.BOOLEAN; import static org.apache.flink.table.api.DataTypes.BYTES; import static org.apache.flink.table.api.DataTypes.FIELD; +import static org.apache.flink.table.api.DataTypes.GEOGRAPHY; import static org.apache.flink.table.api.DataTypes.INT; import static org.apache.flink.table.api.DataTypes.MAP; import static org.apache.flink.table.api.DataTypes.ROW; @@ -260,6 +261,62 @@ Stream getTestSetSpecs() { .testSqlValidationError( "CAST(CreateMultiset(f0) AS BITMAP)", "Cast function cannot convert value of type VARCHAR(2147483647) MULTISET to type BITMAP"), + TestSetSpec.forFunction(BuiltInFunctionDefinitions.CAST, "cast STRING to GEOGRAPHY") + .onFieldsWithData("POINT (0 0)") + .andDataTypes(STRING()) + .testTableApiValidationError( + $("f0").cast(GEOGRAPHY()), + "Unsupported cast from 'STRING' to 'GEOGRAPHY'") + .testSqlValidationError( + "CAST(f0 AS GEOGRAPHY)", + "Cast function cannot convert value of type VARCHAR(2147483647) to type GEOGRAPHY"), + TestSetSpec.forFunction(BuiltInFunctionDefinitions.CAST, "cast BYTES to GEOGRAPHY") + .onFieldsWithData(new byte[] {1, 2, 3}) + .andDataTypes(BYTES()) + .testTableApiValidationError( + $("f0").cast(GEOGRAPHY()), + "Unsupported cast from 'BYTES' to 'GEOGRAPHY'") + .testSqlValidationError( + "CAST(f0 AS GEOGRAPHY)", + "Cast function cannot convert value of type VARBINARY(2147483647) to type GEOGRAPHY"), + TestSetSpec.forFunction( + BuiltInFunctionDefinitions.CAST, "cast VARBINARY to GEOGRAPHY") + .onFieldsWithData(new byte[] {1, 2, 3}) + .andDataTypes(VARBINARY(3)) + .testTableApiValidationError( + $("f0").cast(GEOGRAPHY()), + "Unsupported cast from 'VARBINARY(3)' to 'GEOGRAPHY'") + .testSqlValidationError( + "CAST(f0 AS GEOGRAPHY)", + "Cast function cannot convert value of type VARBINARY(3) to type GEOGRAPHY"), + TestSetSpec.forFunction(BuiltInFunctionDefinitions.CAST, "cast GEOGRAPHY to STRING") + .onFieldsWithData((Object) null) + .andDataTypes(GEOGRAPHY()) + .testTableApiValidationError( + $("f0").cast(STRING()), + "Unsupported cast from 'GEOGRAPHY' to 'STRING'") + .testSqlValidationError( + "CAST(f0 AS STRING)", + "Cast function cannot convert value of type GEOGRAPHY to type VARCHAR(2147483647)"), + TestSetSpec.forFunction(BuiltInFunctionDefinitions.CAST, "cast GEOGRAPHY to BYTES") + .onFieldsWithData((Object) null) + .andDataTypes(GEOGRAPHY()) + .testTableApiValidationError( + $("f0").cast(BYTES()), + "Unsupported cast from 'GEOGRAPHY' to 'BYTES'") + .testSqlValidationError( + "CAST(f0 AS BYTES)", + "Cast function cannot convert value of type GEOGRAPHY to type VARBINARY(2147483647)"), + TestSetSpec.forFunction( + BuiltInFunctionDefinitions.CAST, "cast GEOGRAPHY to VARBINARY") + .onFieldsWithData((Object) null) + .andDataTypes(GEOGRAPHY()) + .testTableApiValidationError( + $("f0").cast(VARBINARY(3)), + "Unsupported cast from 'GEOGRAPHY' to 'VARBINARY(3)'") + .testSqlValidationError( + "CAST(f0 AS VARBINARY(3))", + "Cast function cannot convert value of type GEOGRAPHY to type VARBINARY(3)"), TestSetSpec.forFunction(BuiltInFunctionDefinitions.CAST, "cast RAW to STRING") .onFieldsWithData("2020-11-11T18:08:01.123") .andDataTypes(STRING()) diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/GeographyAccessorFunctionsITCase.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/GeographyAccessorFunctionsITCase.java new file mode 100644 index 00000000000000..a4f40e0d54f8c9 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/GeographyAccessorFunctionsITCase.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.functions; + +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.functions.BuiltInFunctionDefinitions; + +import java.util.stream.Stream; + +/** Tests for GEOGRAPHY accessor functions. */ +class GeographyAccessorFunctionsITCase extends BuiltInFunctionTestBase { + + private static final byte[] POINT_WKB = + new byte[] { + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, (byte) 0xF0, 0x3F, 0, 0, 0, 0, 0, 0, 0, 0x40 + }; + + @Override + Stream getTestSetSpecs() { + return Stream.of(stAsTextCases(), stAsWkbCases()); + } + + private TestSetSpec stAsTextCases() { + return TestSetSpec.forFunction(BuiltInFunctionDefinitions.ST_ASTEXT) + .onFieldsWithData(null, "POINT (1 2)", "POINT EMPTY") + .andDataTypes(DataTypes.STRING(), DataTypes.STRING().notNull(), DataTypes.STRING()) + .testSqlResult("ST_ASTEXT(ST_GEOGFROMTEXT(f0))", null, DataTypes.STRING()) + .testSqlResult( + "ST_ASTEXT(ST_GEOGFROMTEXT(f1))", + "POINT (1 2)", + DataTypes.STRING().notNull()) + .testSqlResult("ST_ASTEXT(ST_GEOGFROMTEXT(f2))", "POINT EMPTY", DataTypes.STRING()); + } + + private TestSetSpec stAsWkbCases() { + return TestSetSpec.forFunction(BuiltInFunctionDefinitions.ST_ASWKB) + .onFieldsWithData(null, POINT_WKB) + .andDataTypes(DataTypes.BYTES(), DataTypes.BYTES().notNull()) + .testSqlResult("ST_ASWKB(ST_GEOGFROMWKB(f0))", null, DataTypes.BYTES()) + .testSqlResult( + "ST_ASWKB(ST_GEOGFROMWKB(f1))", POINT_WKB, DataTypes.BYTES().notNull()); + } +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/GeographyConstructorFunctionsITCase.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/GeographyConstructorFunctionsITCase.java new file mode 100644 index 00000000000000..0788a87ffa0050 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/GeographyConstructorFunctionsITCase.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.functions; + +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.TableRuntimeException; +import org.apache.flink.table.functions.BuiltInFunctionDefinitions; + +import java.util.stream.Stream; + +/** Tests for GEOGRAPHY constructor functions. */ +class GeographyConstructorFunctionsITCase extends BuiltInFunctionTestBase { + + private static final byte[] POINT_WKB = + new byte[] { + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, (byte) 0xF0, 0x3F, 0, 0, 0, 0, 0, 0, 0, 0x40 + }; + + @Override + Stream getTestSetSpecs() { + return Stream.of(stGeogFromTextCases(), stGeogFromWkbCases(), runtimeErrorCases()); + } + + private TestSetSpec stGeogFromTextCases() { + return TestSetSpec.forFunction(BuiltInFunctionDefinitions.ST_GEOGFROMTEXT) + .onFieldsWithData(null, "POINT (1 2)", "POINT EMPTY") + .andDataTypes(DataTypes.STRING(), DataTypes.STRING().notNull(), DataTypes.STRING()) + .testSqlResult("ST_GEOGFROMTEXT(f0)", null, DataTypes.GEOGRAPHY()) + .testSqlResult("ST_GEOGFROMTEXT(f1)", DataTypes.GEOGRAPHY().notNull()) + .testSqlResult("ST_GEOGFROMTEXT(f2)", DataTypes.GEOGRAPHY()); + } + + private TestSetSpec stGeogFromWkbCases() { + return TestSetSpec.forFunction(BuiltInFunctionDefinitions.ST_GEOGFROMWKB) + .onFieldsWithData(null, POINT_WKB) + .andDataTypes(DataTypes.BYTES(), DataTypes.BYTES().notNull()) + .testSqlResult("ST_GEOGFROMWKB(f0)", null, DataTypes.GEOGRAPHY()) + .testSqlResult("ST_GEOGFROMWKB(f1)", DataTypes.GEOGRAPHY().notNull()); + } + + private TestSetSpec runtimeErrorCases() { + return TestSetSpec.forFunction(BuiltInFunctionDefinitions.ST_GEOGFROMTEXT, "Runtime errors") + .onFieldsWithData( + "not wkt", "POINT Z (1 2 3)", "POINT (181 2)", new byte[] {1, 1, 0, 0}) + .andDataTypes( + DataTypes.STRING(), + DataTypes.STRING(), + DataTypes.STRING(), + DataTypes.BYTES()) + .testSqlRuntimeError( + "ST_GEOGFROMTEXT(f0)", + TableRuntimeException.class, + "Invalid GEOGRAPHY WKT.") + .testSqlRuntimeError( + "ST_GEOGFROMTEXT(f1)", + TableRuntimeException.class, + "Only 2D coordinates are supported") + .testSqlRuntimeError( + "ST_GEOGFROMTEXT(f2)", + TableRuntimeException.class, + "Expected range is [-180, 180]") + .testSqlRuntimeError( + "ST_GEOGFROMWKB(f3)", + TableRuntimeException.class, + "Invalid GEOGRAPHY WKB."); + } +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/serde/DataTypeJsonSerdeTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/serde/DataTypeJsonSerdeTest.java index bee8c249624171..0fbab22e154b77 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/serde/DataTypeJsonSerdeTest.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/serde/DataTypeJsonSerdeTest.java @@ -62,6 +62,7 @@ private static Stream testDataTypeSerde() { DataTypes.VARIANT(), DataTypes.VARIANT().notNull(), DataTypes.ARRAY(DataTypes.VARIANT()), + DataTypes.GEOGRAPHY(), DataTypes.ROW( DataTypes.TIMESTAMP_LTZ(3).toInternal(), DataTypes.TIMESTAMP_LTZ(9).bridgedTo(long.class), diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/serde/LogicalTypeJsonSerdeTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/serde/LogicalTypeJsonSerdeTest.java index c838a6ca08add1..be0d487c652ca3 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/serde/LogicalTypeJsonSerdeTest.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/serde/LogicalTypeJsonSerdeTest.java @@ -43,6 +43,7 @@ import org.apache.flink.table.types.logical.DescriptorType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; @@ -273,6 +274,7 @@ private static List testLogicalTypeSerde() { new MultisetType(new VariantType()), new MapType(new VarCharType(5), new VariantType()), RowType.of(new VariantType(), new VariantType(false)), + new GeographyType(), RowType.of(new BigIntType(), new IntType(false), new VarCharType(200)), RowType.of( new LogicalType[] { diff --git a/flink-table/flink-table-runtime/pom.xml b/flink-table/flink-table-runtime/pom.xml index 9958fc66f1b933..65b5609969b00d 100644 --- a/flink-table/flink-table-runtime/pom.xml +++ b/flink-table/flink-table-runtime/pom.xml @@ -78,6 +78,13 @@ under the License. ${flink.markBundledAsOptional} + + org.locationtech.jts + jts-core + 1.19.0 + ${flink.markBundledAsOptional} + + org.apache.flink @@ -186,6 +193,7 @@ under the License. org.codehaus.janino:* org.apache.flink:flink-table-code-splitter org.apache.flink:flink-table-type-utils + org.locationtech.jts:jts-core diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/data/BoxedWrapperRowData.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/data/BoxedWrapperRowData.java index d5e7f361d7fa98..33030f2e35775c 100644 --- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/data/BoxedWrapperRowData.java +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/data/BoxedWrapperRowData.java @@ -154,6 +154,11 @@ public Bitmap getBitmap(int pos) { return (Bitmap) this.fields[pos]; } + @Override + public GeographyData getGeography(int pos) { + return (GeographyData) this.fields[pos]; + } + @Override public void setNullAt(int pos) { this.fields[pos] = null; diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/data/util/DataFormatConverters.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/data/util/DataFormatConverters.java index 756ca6293c8676..166aa582d94570 100644 --- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/data/util/DataFormatConverters.java +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/data/util/DataFormatConverters.java @@ -35,6 +35,7 @@ import org.apache.flink.table.data.GenericArrayData; import org.apache.flink.table.data.GenericMapData; import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.MapData; import org.apache.flink.table.data.RawValueData; import org.apache.flink.table.data.RowData; @@ -590,6 +591,21 @@ StringData toExternalImpl(RowData row, int column) { } } + /** Converter for GeographyData. */ + public static final class GeographyConverter extends IdentityConverter { + + private static final long serialVersionUID = -2785143344060029173L; + + public static final GeographyConverter INSTANCE = new GeographyConverter(); + + private GeographyConverter() {} + + @Override + GeographyData toExternalImpl(RowData row, int column) { + return row.getGeography(column); + } + } + /** Converter for ArrayData. */ public static final class ArrayDataConverter extends IdentityConverter { diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/GeographyConversionUtils.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/GeographyConversionUtils.java new file mode 100644 index 00000000000000..06c2f9736190f9 --- /dev/null +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/GeographyConversionUtils.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.functions.scalar; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.api.TableRuntimeException; +import org.apache.flink.table.data.GeographyData; +import org.apache.flink.table.data.StringData; + +import org.locationtech.jts.geom.CoordinateSequence; +import org.locationtech.jts.geom.CoordinateSequenceFilter; +import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.io.ByteOrderValues; +import org.locationtech.jts.io.ParseException; +import org.locationtech.jts.io.WKBReader; +import org.locationtech.jts.io.WKBWriter; +import org.locationtech.jts.io.WKTReader; +import org.locationtech.jts.io.WKTWriter; + +import java.util.regex.Pattern; + +/** Utilities for converting GEOGRAPHY values to and from portable OGC encodings. */ +@Internal +final class GeographyConversionUtils { + + private static final Pattern WKT_DIMENSION_TOKEN = + Pattern.compile( + "\\b(POINT|LINESTRING|POLYGON|MULTIPOINT|MULTILINESTRING|MULTIPOLYGON|GEOMETRYCOLLECTION)\\s+(ZM|Z|M)\\b", + Pattern.CASE_INSENSITIVE); + private static final WKTReader WKT_READER = createWktReader(); + private static final WKBReader WKB_READER = new WKBReader(); + private static final WKBWriter WKB_WRITER = + new WKBWriter(2, ByteOrderValues.LITTLE_ENDIAN, false); + private static final WKTWriter WKT_WRITER = new WKTWriter(2); + + private GeographyConversionUtils() {} + + static GeographyData fromWkt(StringData text) { + if (text == null) { + return null; + } + + final String wkt = text.toString(); + if (WKT_DIMENSION_TOKEN.matcher(wkt).find()) { + throw new TableRuntimeException( + "Invalid GEOGRAPHY WKT. Only 2D coordinates are supported."); + } + + final Geometry geometry; + try { + geometry = WKT_READER.read(wkt); + } catch (ParseException | IllegalArgumentException e) { + throw new TableRuntimeException("Invalid GEOGRAPHY WKT.", e); + } + validateGeometry(geometry); + return fromValidatedGeometry(geometry); + } + + static GeographyData fromWkb(byte[] bytes) { + if (bytes == null) { + return null; + } + + final GeographyData geography; + try { + geography = GeographyData.fromBytes(bytes); + } catch (TableRuntimeException e) { + throw new TableRuntimeException("Invalid GEOGRAPHY WKB.", e); + } + + validateGeometry(readGeometry(geography)); + return geography; + } + + static StringData asText(GeographyData geography) { + if (geography == null) { + return null; + } + + return StringData.fromString(WKT_WRITER.write(readGeometry(geography))); + } + + static byte[] asWkb(GeographyData geography) { + if (geography == null) { + return null; + } + + return geography.toBytes(); + } + + private static GeographyData fromValidatedGeometry(Geometry geometry) { + try { + return GeographyData.fromBytes(WKB_WRITER.write(geometry)); + } catch (TableRuntimeException e) { + throw new TableRuntimeException("Invalid GEOGRAPHY WKT.", e); + } + } + + private static Geometry readGeometry(GeographyData geography) { + try { + return WKB_READER.read(geography.toBytes()); + } catch (ParseException e) { + throw new TableRuntimeException("Invalid GEOGRAPHY WKB.", e); + } + } + + private static WKTReader createWktReader() { + final WKTReader reader = new WKTReader(); + reader.setIsOldJtsCoordinateSyntaxAllowed(false); + return reader; + } + + private static void validateGeometry(Geometry geometry) { + validateSubtype(geometry); + if (!geometry.isEmpty()) { + geometry.apply(new Crs84CoordinateValidator()); + } + } + + private static void validateSubtype(Geometry geometry) { + switch (geometry.getGeometryType()) { + case Geometry.TYPENAME_POINT: + case Geometry.TYPENAME_LINESTRING: + case Geometry.TYPENAME_POLYGON: + case Geometry.TYPENAME_MULTIPOINT: + case Geometry.TYPENAME_MULTILINESTRING: + case Geometry.TYPENAME_MULTIPOLYGON: + case Geometry.TYPENAME_GEOMETRYCOLLECTION: + return; + default: + throw new TableRuntimeException( + "Unsupported GEOGRAPHY subtype: " + geometry.getGeometryType() + "."); + } + } + + private static final class Crs84CoordinateValidator implements CoordinateSequenceFilter { + + @Override + public void filter(CoordinateSequence sequence, int i) { + if (sequence.hasZ() || sequence.hasM()) { + throw new TableRuntimeException( + "Invalid GEOGRAPHY coordinates. Only 2D coordinates are supported."); + } + + final double longitude = sequence.getX(i); + final double latitude = sequence.getY(i); + if (longitude < -180D || longitude > 180D) { + throw new TableRuntimeException( + String.format( + "Invalid GEOGRAPHY longitude %.12f. Expected range is [-180, 180].", + longitude)); + } + if (latitude < -90D || latitude > 90D) { + throw new TableRuntimeException( + String.format( + "Invalid GEOGRAPHY latitude %.12f. Expected range is [-90, 90].", + latitude)); + } + } + + @Override + public boolean isDone() { + return false; + } + + @Override + public boolean isGeometryChanged() { + return false; + } + } +} diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/StAsTextFunction.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/StAsTextFunction.java new file mode 100644 index 00000000000000..17940f9ebc18a6 --- /dev/null +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/StAsTextFunction.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.functions.scalar; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.data.GeographyData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.functions.BuiltInFunctionDefinitions; +import org.apache.flink.table.functions.SpecializedFunction.SpecializedContext; + +import javax.annotation.Nullable; + +/** Implementation of {@link BuiltInFunctionDefinitions#ST_ASTEXT}. */ +@Internal +public class StAsTextFunction extends BuiltInScalarFunction { + + public StAsTextFunction(SpecializedContext context) { + super(BuiltInFunctionDefinitions.ST_ASTEXT, context); + } + + public @Nullable StringData eval(@Nullable GeographyData geography) { + return GeographyConversionUtils.asText(geography); + } +} diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/StAsWkbFunction.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/StAsWkbFunction.java new file mode 100644 index 00000000000000..27baaac1637eb9 --- /dev/null +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/StAsWkbFunction.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.functions.scalar; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.data.GeographyData; +import org.apache.flink.table.functions.BuiltInFunctionDefinitions; +import org.apache.flink.table.functions.SpecializedFunction.SpecializedContext; + +import javax.annotation.Nullable; + +/** Implementation of {@link BuiltInFunctionDefinitions#ST_ASWKB}. */ +@Internal +public class StAsWkbFunction extends BuiltInScalarFunction { + + public StAsWkbFunction(SpecializedContext context) { + super(BuiltInFunctionDefinitions.ST_ASWKB, context); + } + + public @Nullable byte[] eval(@Nullable GeographyData geography) { + return GeographyConversionUtils.asWkb(geography); + } +} diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/StGeogFromTextFunction.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/StGeogFromTextFunction.java new file mode 100644 index 00000000000000..4781ad2d5a7ead --- /dev/null +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/StGeogFromTextFunction.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.functions.scalar; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.data.GeographyData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.functions.BuiltInFunctionDefinitions; +import org.apache.flink.table.functions.SpecializedFunction.SpecializedContext; + +import javax.annotation.Nullable; + +/** Implementation of {@link BuiltInFunctionDefinitions#ST_GEOGFROMTEXT}. */ +@Internal +public class StGeogFromTextFunction extends BuiltInScalarFunction { + + public StGeogFromTextFunction(SpecializedContext context) { + super(BuiltInFunctionDefinitions.ST_GEOGFROMTEXT, context); + } + + public @Nullable GeographyData eval(@Nullable StringData text) { + return GeographyConversionUtils.fromWkt(text); + } +} diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/StGeogFromWkbFunction.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/StGeogFromWkbFunction.java new file mode 100644 index 00000000000000..c3a17ac96f2064 --- /dev/null +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/StGeogFromWkbFunction.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.functions.scalar; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.data.GeographyData; +import org.apache.flink.table.functions.BuiltInFunctionDefinitions; +import org.apache.flink.table.functions.SpecializedFunction.SpecializedContext; + +import javax.annotation.Nullable; + +/** Implementation of {@link BuiltInFunctionDefinitions#ST_GEOGFROMWKB}. */ +@Internal +public class StGeogFromWkbFunction extends BuiltInScalarFunction { + + public StGeogFromWkbFunction(SpecializedContext context) { + super(BuiltInFunctionDefinitions.ST_GEOGFROMWKB, context); + } + + public @Nullable GeographyData eval(@Nullable byte[] bytes) { + return GeographyConversionUtils.fromWkb(bytes); + } +} diff --git a/flink-table/flink-table-runtime/src/main/resources/META-INF/NOTICE b/flink-table/flink-table-runtime/src/main/resources/META-INF/NOTICE index 0a2b4b723f1e96..a3e63d8fd221a2 100644 --- a/flink-table/flink-table-runtime/src/main/resources/META-INF/NOTICE +++ b/flink-table/flink-table-runtime/src/main/resources/META-INF/NOTICE @@ -9,3 +9,7 @@ This project bundles the following dependencies under the Apache Software Licens - com.jayway.jsonpath:json-path:2.7.0 - org.codehaus.janino:janino:3.1.12 - org.codehaus.janino:commons-compiler:3.1.12 + +This project bundles the following dependencies under the Eclipse Public License - v 2.0 (https://www.eclipse.org/org/documents/epl-2.0/EPL-2.0.txt) + +- org.locationtech.jts:jts-core:1.19.0 diff --git a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/data/RowDataTest.java b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/data/RowDataTest.java index 787e54957c7490..37e5d492a55774 100644 --- a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/data/RowDataTest.java +++ b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/data/RowDataTest.java @@ -33,11 +33,13 @@ import org.apache.flink.table.types.logical.ArrayType; import org.apache.flink.table.types.logical.BigIntType; import org.apache.flink.table.types.logical.BinaryType; +import org.apache.flink.table.types.logical.BitmapType; import org.apache.flink.table.types.logical.BooleanType; import org.apache.flink.table.types.logical.CharType; import org.apache.flink.table.types.logical.DecimalType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.MapType; import org.apache.flink.table.types.logical.MultisetType; @@ -63,7 +65,11 @@ /** Test for {@link RowData}s. */ class RowDataTest { - private static final int NUM_FIELDS = 19; + private static final int NUM_FIELDS = 20; + private static final byte[] POINT_WKB = + new byte[] { + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, (byte) 0xF0, 0x3F, 0, 0, 0, 0, 0, 0, 0, 0x40 + }; private StringData str; private RawValueData generic; @@ -77,6 +83,7 @@ class RowDataTest { private TimestampData timestamp1; private TimestampData timestamp2; private Bitmap bitmap; + private GeographyData geography; @BeforeEach void before() { @@ -105,6 +112,7 @@ void before() { timestamp2 = TimestampData.fromLocalDateTime(LocalDateTime.of(1969, 1, 1, 0, 0, 0, 123456789)); bitmap = Bitmap.fromArray(new int[] {1, 2, 3}); + geography = GeographyData.fromBytes(POINT_WKB); } @Test @@ -152,6 +160,7 @@ private BinaryRowData getBinaryRow() { writer.writeTimestamp(16, timestamp1, 3); writer.writeTimestamp(17, timestamp2, 9); writer.writeBitmap(18, bitmap); + writer.writeGeography(19, geography); return row; } @@ -177,6 +186,7 @@ void testGenericRow() { row.setField(16, timestamp1); row.setField(17, timestamp2); row.setField(18, bitmap); + row.setField(19, geography); testGetters(row); } @@ -201,6 +211,7 @@ public void testBoxedWrapperRow() { row.setNonPrimitiveValue(16, timestamp1); row.setNonPrimitiveValue(17, timestamp2); row.setNonPrimitiveValue(18, bitmap); + row.setNonPrimitiveValue(19, geography); testGetters(row); testSetters(row); } @@ -230,6 +241,7 @@ public void testJoinedRow() { row2.setField(11, timestamp1); row2.setField(12, timestamp2); row2.setField(13, bitmap); + row2.setField(14, geography); testGetters(new JoinedRowData(row1, row2)); } @@ -277,6 +289,14 @@ void testFieldGetters() { .isEqualTo(timestamp1); assertThat(RowData.createFieldGetter(new TimestampType(9), 17).getFieldOrNull(row)) .isEqualTo(timestamp2); + assertThat(RowData.createFieldGetter(new BitmapType(), 18).getFieldOrNull(row)) + .isEqualTo(bitmap); + assertThat( + ((GeographyData) + RowData.createFieldGetter(new GeographyType(), 19) + .getFieldOrNull(row)) + .toBytes()) + .isEqualTo(geography.toBytes()); } @Test @@ -355,6 +375,10 @@ private void testFieldGettersWithNull(boolean nullable) { RowData.createFieldGetter(new TimestampType(nullable, 9), 17) .getFieldOrNull(row)) .isNull(); + assertThat(RowData.createFieldGetter(new BitmapType(nullable), 18).getFieldOrNull(row)) + .isNull(); + assertThat(RowData.createFieldGetter(new GeographyType(nullable), 19).getFieldOrNull(row)) + .isNull(); } private void testGetters(RowData row) { @@ -385,6 +409,7 @@ private void testGetters(RowData row) { assertThat(row.getTimestamp(16, 3)).isEqualTo(timestamp1); assertThat(row.getTimestamp(17, 9)).isEqualTo(timestamp2); assertThat(row.getBitmap(18)).isEqualTo(bitmap); + assertThat(row.getGeography(19).toBytes()).isEqualTo(geography.toBytes()); } private void testSetters(RowData row) { @@ -435,7 +460,7 @@ private void testSetters(RowData row) { } private static BinaryRowData getNullBinaryRow() { - BinaryRowData row = new BinaryRowData(18); + BinaryRowData row = new BinaryRowData(NUM_FIELDS); BinaryRowWriter binaryRowWriter = new BinaryRowWriter(row); for (int i = 0; i < row.getArity(); i++) { binaryRowWriter.setNullAt(i); diff --git a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/functions/scalar/GeographyAccessorFunctionsTest.java b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/functions/scalar/GeographyAccessorFunctionsTest.java new file mode 100644 index 00000000000000..d25dfd265fc128 --- /dev/null +++ b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/functions/scalar/GeographyAccessorFunctionsTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.functions.scalar; + +import org.apache.flink.table.data.GeographyData; +import org.apache.flink.table.data.StringData; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for GEOGRAPHY accessor conversion utilities. */ +class GeographyAccessorFunctionsTest { + + private static final byte[] POINT_WKB = + new byte[] { + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, (byte) 0xF0, 0x3F, 0, 0, 0, 0, 0, 0, 0, 0x40 + }; + + @Test + void testAsTextReturnsWkt() { + final GeographyData geography = GeographyData.fromBytes(POINT_WKB); + + assertThat(GeographyConversionUtils.asText(geography)) + .isEqualTo(StringData.fromString("POINT (1 2)")); + } + + @Test + void testAsTextSupportsEmptyGeometry() { + final GeographyData geography = + GeographyConversionUtils.fromWkt(StringData.fromString("POINT EMPTY")); + + assertThat(GeographyConversionUtils.asText(geography)) + .isEqualTo(StringData.fromString("POINT EMPTY")); + } + + @Test + void testAsWkbReturnsRawIsoWkb() { + final GeographyData geography = GeographyData.fromBytes(POINT_WKB); + + assertThat(GeographyConversionUtils.asWkb(geography)) + .hasSize(POINT_WKB.length) + .isEqualTo(POINT_WKB); + } + + @Test + void testAccessorsPropagateNull() { + assertThat(GeographyConversionUtils.asText(null)).isNull(); + assertThat(GeographyConversionUtils.asWkb(null)).isNull(); + } +} diff --git a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/functions/scalar/GeographyConstructorFunctionsTest.java b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/functions/scalar/GeographyConstructorFunctionsTest.java new file mode 100644 index 00000000000000..af64f32329f810 --- /dev/null +++ b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/functions/scalar/GeographyConstructorFunctionsTest.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.functions.scalar; + +import org.apache.flink.table.api.TableRuntimeException; +import org.apache.flink.table.data.GeographyData; +import org.apache.flink.table.data.StringData; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for GEOGRAPHY constructor conversion utilities. */ +class GeographyConstructorFunctionsTest { + + private static final byte[] POINT_WKB = + new byte[] { + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, (byte) 0xF0, 0x3F, 0, 0, 0, 0, 0, 0, 0, 0x40 + }; + + @Test + void testFromWktCreatesIsoWkb() { + final GeographyData geography = + GeographyConversionUtils.fromWkt(StringData.fromString("POINT (1 2)")); + + assertThat(geography.subtypeId()).isEqualTo(GeographyData.POINT); + assertThat(geography.toBytes()).isEqualTo(POINT_WKB); + } + + @Test + void testFromWktAllowsEmptyGeometry() { + final GeographyData geography = + GeographyConversionUtils.fromWkt(StringData.fromString("POINT EMPTY")); + + assertThat(geography.subtypeId()).isEqualTo(GeographyData.POINT); + } + + @Test + void testFromWkbKeepsRawIsoWkb() { + final GeographyData geography = GeographyConversionUtils.fromWkb(POINT_WKB); + + assertThat(geography.toBytes()).isEqualTo(POINT_WKB); + } + + @Test + void testConstructorsPropagateNull() { + assertThat(GeographyConversionUtils.fromWkt(null)).isNull(); + assertThat(GeographyConversionUtils.fromWkb(null)).isNull(); + } + + @Test + void testFromWktRejectsMalformedInput() { + assertThatThrownBy(() -> GeographyConversionUtils.fromWkt(StringData.fromString("not wkt"))) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Invalid GEOGRAPHY WKT."); + } + + @Test + void testFromWktRejectsThreeDimensionalInput() { + assertThatThrownBy( + () -> + GeographyConversionUtils.fromWkt( + StringData.fromString("POINT Z (1 2 3)"))) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Only 2D coordinates are supported"); + } + + @Test + void testFromWktRejectsCoordinatesOutsideCrs84Range() { + assertThatThrownBy( + () -> + GeographyConversionUtils.fromWkt( + StringData.fromString("POINT (181 2)"))) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Expected range is [-180, 180]"); + + assertThatThrownBy( + () -> + GeographyConversionUtils.fromWkt( + StringData.fromString("POINT (1 91)"))) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Expected range is [-90, 90]"); + } + + @Test + void testFromWkbRejectsMalformedInput() { + assertThatThrownBy(() -> GeographyConversionUtils.fromWkb(new byte[] {1, 1, 0, 0})) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Invalid GEOGRAPHY WKB."); + } +} diff --git a/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/conversion/DataStructureConverters.java b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/conversion/DataStructureConverters.java index f01a1aca779da7..3123edcb3b8442 100644 --- a/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/conversion/DataStructureConverters.java +++ b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/conversion/DataStructureConverters.java @@ -22,11 +22,13 @@ import org.apache.flink.table.api.TableException; import org.apache.flink.table.data.ArrayData; import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.MapData; import org.apache.flink.table.data.RawValueData; import org.apache.flink.table.data.RowData; import org.apache.flink.table.data.StringData; import org.apache.flink.table.data.TimestampData; +import org.apache.flink.table.data.binary.BinaryGeographyData; import org.apache.flink.table.types.DataType; import org.apache.flink.table.types.logical.LogicalType; import org.apache.flink.table.types.logical.LogicalTypeRoot; @@ -200,6 +202,8 @@ public final class DataStructureConverters { putConverter(LogicalTypeRoot.VARIANT, Variant.class, identity()); putConverter(LogicalTypeRoot.BITMAP, Bitmap.class, constructor(BitmapBitmapConverter::new)); putConverter(LogicalTypeRoot.BITMAP, RoaringBitmapData.class, identity()); + putConverter(LogicalTypeRoot.GEOGRAPHY, GeographyData.class, identity()); + putConverter(LogicalTypeRoot.GEOGRAPHY, BinaryGeographyData.class, identity()); } /** Returns a converter for the given {@link DataType}. */ diff --git a/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/writer/AbstractBinaryWriter.java b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/writer/AbstractBinaryWriter.java index 73f817e64815eb..0564d0b0cc634e 100644 --- a/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/writer/AbstractBinaryWriter.java +++ b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/writer/AbstractBinaryWriter.java @@ -24,6 +24,7 @@ import org.apache.flink.core.memory.MemorySegmentFactory; import org.apache.flink.table.data.ArrayData; import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.MapData; import org.apache.flink.table.data.RawValueData; import org.apache.flink.table.data.RowData; @@ -141,6 +142,11 @@ public void writeBitmap(int pos, Bitmap bitmap) { writeBytesToVarLenPart(pos, bytes, bytes.length); } + @Override + public void writeGeography(int pos, GeographyData geography) { + writeBinary(pos, geography.toBytes()); + } + private DataOutputViewStreamWrapper getOutputView() { if (outputView == null) { outputView = new DataOutputViewStreamWrapper(new BinaryRowWriterOutputView()); diff --git a/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/writer/BinaryArrayWriter.java b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/writer/BinaryArrayWriter.java index 857c2ecd42466f..68540b9fc93e25 100644 --- a/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/writer/BinaryArrayWriter.java +++ b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/writer/BinaryArrayWriter.java @@ -247,6 +247,7 @@ public static NullSetter createNullSetter(LogicalType elementType) { case RAW: case VARIANT: case BITMAP: + case GEOGRAPHY: return BinaryArrayWriter::setNullLong; case BOOLEAN: return BinaryArrayWriter::setNullBoolean; diff --git a/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/writer/BinaryWriter.java b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/writer/BinaryWriter.java index 4db8977ccd2f0b..482540b1d53312 100644 --- a/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/writer/BinaryWriter.java +++ b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/writer/BinaryWriter.java @@ -21,6 +21,7 @@ import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.table.data.ArrayData; import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.MapData; import org.apache.flink.table.data.RawValueData; import org.apache.flink.table.data.RowData; @@ -91,6 +92,8 @@ public interface BinaryWriter { void writeBitmap(int pos, Bitmap bitmap); + void writeGeography(int pos, GeographyData geography); + /** Finally, complete write to set real size to binary. */ void complete(); @@ -174,6 +177,9 @@ static void write( case BITMAP: writer.writeBitmap(pos, (Bitmap) o); break; + case GEOGRAPHY: + writer.writeGeography(pos, (GeographyData) o); + break; default: throw new UnsupportedOperationException("Not support type: " + type); } @@ -253,6 +259,8 @@ static ValueSetter createValueSetter(LogicalType elementType) { return (writer, pos, value) -> writer.writeVariant(pos, (Variant) value); case BITMAP: return (writer, pos, value) -> writer.writeBitmap(pos, (Bitmap) value); + case GEOGRAPHY: + return (writer, pos, value) -> writer.writeGeography(pos, (GeographyData) value); case NULL: case SYMBOL: case UNRESOLVED: diff --git a/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/runtime/typeutils/GeographyTypeSerializer.java b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/runtime/typeutils/GeographyTypeSerializer.java new file mode 100644 index 00000000000000..3e54f24009eb82 --- /dev/null +++ b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/runtime/typeutils/GeographyTypeSerializer.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.typeutils; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; +import org.apache.flink.api.common.typeutils.base.TypeSerializerSingleton; +import org.apache.flink.core.memory.DataInputView; +import org.apache.flink.core.memory.DataOutputView; +import org.apache.flink.table.data.GeographyData; + +import java.io.IOException; + +/** + * Serializer for {@link GeographyData} values of {@link + * org.apache.flink.table.types.logical.GeographyType}. + */ +@Internal +public final class GeographyTypeSerializer extends TypeSerializerSingleton { + + private static final long serialVersionUID = 1L; + + private static final int FORMAT_VERSION = 1; + + private static final byte[] EMPTY_GEOMETRY_COLLECTION = + new byte[] {1, GeographyData.GEOMETRY_COLLECTION, 0, 0, 0, 0, 0, 0, 0}; + + public static final GeographyTypeSerializer INSTANCE = new GeographyTypeSerializer(); + + private GeographyTypeSerializer() {} + + @Override + public boolean isImmutableType() { + return true; + } + + @Override + public GeographyData createInstance() { + return GeographyData.fromBytes(EMPTY_GEOMETRY_COLLECTION); + } + + @Override + public GeographyData copy(GeographyData from) { + return GeographyData.fromBytes(from.toBytes()); + } + + @Override + public GeographyData copy(GeographyData from, GeographyData reuse) { + return copy(from); + } + + @Override + public int getLength() { + return -1; + } + + @Override + public void serialize(GeographyData record, DataOutputView target) throws IOException { + final byte[] bytes = record.toBytes(); + target.writeByte(FORMAT_VERSION); + target.writeInt(bytes.length); + target.write(bytes); + } + + @Override + public GeographyData deserialize(DataInputView source) throws IOException { + readFormatVersion(source); + final int length = readPayloadLength(source); + final byte[] bytes = new byte[length]; + source.readFully(bytes); + return GeographyData.fromBytes(bytes); + } + + @Override + public GeographyData deserialize(GeographyData reuse, DataInputView source) throws IOException { + return deserialize(source); + } + + @Override + public void copy(DataInputView source, DataOutputView target) throws IOException { + final int version = readFormatVersion(source); + final int length = readPayloadLength(source); + target.writeByte(version); + target.writeInt(length); + target.write(source, length); + } + + @Override + public TypeSerializerSnapshot snapshotConfiguration() { + return new GeographyTypeSerializerSnapshot(); + } + + private static int readFormatVersion(DataInputView source) throws IOException { + final int version = source.readUnsignedByte(); + if (version != FORMAT_VERSION) { + throw new IOException( + String.format( + "Unsupported GEOGRAPHY serializer format version %d. Expected %d.", + version, FORMAT_VERSION)); + } + return version; + } + + private static int readPayloadLength(DataInputView source) throws IOException { + final int length = source.readInt(); + if (length < 0) { + throw new IOException(String.format("Invalid GEOGRAPHY payload length %d.", length)); + } + return length; + } +} diff --git a/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/runtime/typeutils/GeographyTypeSerializerSnapshot.java b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/runtime/typeutils/GeographyTypeSerializerSnapshot.java new file mode 100644 index 00000000000000..a3159016479899 --- /dev/null +++ b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/runtime/typeutils/GeographyTypeSerializerSnapshot.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.typeutils; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility; +import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; +import org.apache.flink.core.memory.DataInputView; +import org.apache.flink.core.memory.DataOutputView; +import org.apache.flink.table.data.GeographyData; + +import java.io.IOException; + +/** Serializer snapshot for {@link GeographyTypeSerializer}. */ +@Internal +public final class GeographyTypeSerializerSnapshot + implements TypeSerializerSnapshot { + + private static final int CURRENT_VERSION = 1; + + @Override + public int getCurrentVersion() { + return CURRENT_VERSION; + } + + @Override + public void writeSnapshot(DataOutputView out) throws IOException {} + + @Override + public void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader) + throws IOException {} + + @Override + public TypeSerializer restoreSerializer() { + return GeographyTypeSerializer.INSTANCE; + } + + @Override + public TypeSerializerSchemaCompatibility resolveSchemaCompatibility( + TypeSerializerSnapshot oldSerializerSnapshot) { + if (oldSerializerSnapshot instanceof GeographyTypeSerializerSnapshot) { + return TypeSerializerSchemaCompatibility.compatibleAsIs(); + } + return TypeSerializerSchemaCompatibility.incompatible(); + } +} diff --git a/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/runtime/typeutils/InternalSerializers.java b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/runtime/typeutils/InternalSerializers.java index 4334591cbd15b3..478b6c4a2a8f02 100644 --- a/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/runtime/typeutils/InternalSerializers.java +++ b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/runtime/typeutils/InternalSerializers.java @@ -130,6 +130,8 @@ private static TypeSerializer createInternal(LogicalType type) { return VariantSerializer.INSTANCE; case BITMAP: return BitmapSerializer.INSTANCE; + case GEOGRAPHY: + return GeographyTypeSerializer.INSTANCE; case NULL: case SYMBOL: case UNRESOLVED: diff --git a/flink-table/flink-table-type-utils/src/test/java/org/apache/flink/table/runtime/typeutils/GeographyTypeSerializerTest.java b/flink-table/flink-table-type-utils/src/test/java/org/apache/flink/table/runtime/typeutils/GeographyTypeSerializerTest.java new file mode 100644 index 00000000000000..8a695c5c020f54 --- /dev/null +++ b/flink-table/flink-table-type-utils/src/test/java/org/apache/flink/table/runtime/typeutils/GeographyTypeSerializerTest.java @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.typeutils; + +import org.apache.flink.api.common.typeutils.SerializerTestBase; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; +import org.apache.flink.api.common.typeutils.TypeSerializerSnapshotSerializationUtil; +import org.apache.flink.core.memory.DataInputViewStreamWrapper; +import org.apache.flink.core.memory.DataOutputViewStreamWrapper; +import org.apache.flink.table.data.GenericArrayData; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.GeographyData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.binary.BinaryArrayData; +import org.apache.flink.table.data.binary.BinaryRowData; +import org.apache.flink.table.types.logical.GeographyType; + +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link GeographyTypeSerializer}. */ +class GeographyTypeSerializerTest extends SerializerTestBase { + + private static final int FORMAT_VERSION = 1; + + private static final byte[] POINT_WKB = + new byte[] { + 1, GeographyData.POINT, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + + private static final byte[] BIG_ENDIAN_POINT_WKB = + new byte[] { + 0, 0, 0, 0, GeographyData.POINT, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + + @Override + protected TypeSerializer createSerializer() { + return GeographyTypeSerializer.INSTANCE; + } + + @Override + protected int getLength() { + return -1; + } + + @Override + protected Class getTypeClass() { + return GeographyData.class; + } + + @Override + protected GeographyData[] getTestData() { + return new GeographyData[] { + GeographyData.fromBytes(POINT_WKB), + GeographyData.fromBytes(BIG_ENDIAN_POINT_WKB), + GeographyTypeSerializer.INSTANCE.createInstance() + }; + } + + @Override + protected void deepEquals(String message, GeographyData should, GeographyData is) { + assertThat(is.toBytes()).as(message).isEqualTo(should.toBytes()); + } + + @Test + void testInternalSerializerRoundTripsRawWkb() throws Exception { + final TypeSerializer serializer = + InternalSerializers.create(new GeographyType()); + final GeographyData geography = GeographyData.fromBytes(POINT_WKB); + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + serializer.serialize(geography, new DataOutputViewStreamWrapper(bytes)); + final GeographyData deserialized = + serializer.deserialize( + new DataInputViewStreamWrapper( + new ByteArrayInputStream(bytes.toByteArray()))); + + assertThat(deserialized.toBytes()).isEqualTo(POINT_WKB); + assertThat(deserialized.subtypeId()).isEqualTo(GeographyData.POINT); + } + + @Test + void testSerializedFormUsesVersionedLengthEnvelope() throws Exception { + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + GeographyTypeSerializer.INSTANCE.serialize( + GeographyData.fromBytes(POINT_WKB), new DataOutputViewStreamWrapper(bytes)); + final DataInputViewStreamWrapper input = + new DataInputViewStreamWrapper(new ByteArrayInputStream(bytes.toByteArray())); + final byte[] payload = new byte[POINT_WKB.length]; + + assertThat(input.readUnsignedByte()).isEqualTo(FORMAT_VERSION); + assertThat(input.readInt()).isEqualTo(POINT_WKB.length); + input.readFully(payload); + assertThat(payload).isEqualTo(POINT_WKB); + } + + @Test + void testRejectsUnsupportedPayloadVersion() throws Exception { + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + final DataOutputViewStreamWrapper output = new DataOutputViewStreamWrapper(bytes); + output.writeByte(2); + output.writeInt(POINT_WKB.length); + output.write(POINT_WKB); + + assertThatThrownBy( + () -> + GeographyTypeSerializer.INSTANCE.deserialize( + new DataInputViewStreamWrapper( + new ByteArrayInputStream(bytes.toByteArray())))) + .isInstanceOf(IOException.class) + .hasMessageContaining("Unsupported GEOGRAPHY serializer format version 2"); + } + + @Test + void testSnapshotSelfCompatibility() throws Exception { + final TypeSerializerSnapshot snapshot = + GeographyTypeSerializer.INSTANCE.snapshotConfiguration(); + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot( + new DataOutputViewStreamWrapper(bytes), snapshot); + final TypeSerializerSnapshot restoredSnapshot = + TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot( + new DataInputViewStreamWrapper( + new ByteArrayInputStream(bytes.toByteArray())), + getClass().getClassLoader()); + + assertThat( + GeographyTypeSerializer.INSTANCE + .snapshotConfiguration() + .resolveSchemaCompatibility(restoredSnapshot) + .isCompatibleAsIs()) + .isTrue(); + assertThat(restoredSnapshot.restoreSerializer()).isSameAs(GeographyTypeSerializer.INSTANCE); + } + + @Test + void testCopyPreservesRawWkbBytes() { + final GeographyData geography = GeographyData.fromBytes(POINT_WKB); + final GeographyData copied = GeographyTypeSerializer.INSTANCE.copy(geography); + + assertThat(copied).isNotSameAs(geography); + assertThat(copied.toBytes()).isEqualTo(POINT_WKB); + assertThat(copied.subtypeId()).isEqualTo(GeographyData.POINT); + } + + @Test + void testCreateInstanceReturnsValidGeographyData() { + final GeographyData instance = GeographyTypeSerializer.INSTANCE.createInstance(); + + assertThat(instance.subtypeId()).isEqualTo(GeographyData.GEOMETRY_COLLECTION); + assertThat(instance.sizeInBytes()).isEqualTo(9); + } + + @Test + void testRowDataSerializerConvertsGenericRowToBinaryRow() { + final RowDataSerializer serializer = + InternalTypeInfo.ofFields(new GeographyType()).toRowSerializer(); + final GeographyData geography = GeographyData.fromBytes(POINT_WKB); + final GenericRowData row = GenericRowData.of(geography); + + final BinaryRowData binaryRow = serializer.toBinaryRow(row, true); + + assertThat(binaryRow.getGeography(0).toBytes()).isEqualTo(POINT_WKB); + assertThat(binaryRow.getGeography(0).subtypeId()).isEqualTo(GeographyData.POINT); + } + + @Test + void testRowDataSerializerPreservesNullGeography() { + final RowDataSerializer serializer = + InternalTypeInfo.ofFields(new GeographyType()).toRowSerializer(); + final GenericRowData row = GenericRowData.of((Object) null); + + final BinaryRowData binaryRow = serializer.toBinaryRow(row, true); + + assertThat(binaryRow.isNullAt(0)).isTrue(); + } + + @Test + void testArrayDataSerializerConvertsGenericArrayToBinaryArray() { + final ArrayDataSerializer serializer = new ArrayDataSerializer(new GeographyType()); + final GenericArrayData array = + new GenericArrayData(new Object[] {GeographyData.fromBytes(POINT_WKB), null}); + + final BinaryArrayData binaryArray = serializer.toBinaryArray(array); + + assertThat(binaryArray.getGeography(0).toBytes()).isEqualTo(POINT_WKB); + assertThat(binaryArray.getGeography(0).subtypeId()).isEqualTo(GeographyData.POINT); + assertThat(binaryArray.isNullAt(1)).isTrue(); + } +} diff --git a/flink-table/flink-table-type-utils/src/test/java/org/apache/flink/table/runtime/typeutils/RowDataSerializerTest.java b/flink-table/flink-table-type-utils/src/test/java/org/apache/flink/table/runtime/typeutils/RowDataSerializerTest.java index 6d6379ed046965..e5a09b407e45b6 100644 --- a/flink-table/flink-table-type-utils/src/test/java/org/apache/flink/table/runtime/typeutils/RowDataSerializerTest.java +++ b/flink-table/flink-table-type-utils/src/test/java/org/apache/flink/table/runtime/typeutils/RowDataSerializerTest.java @@ -24,6 +24,7 @@ import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer; import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.RawValueData; import org.apache.flink.table.data.RowData; import org.apache.flink.table.data.StringData; @@ -41,6 +42,7 @@ import org.apache.flink.table.types.logical.DecimalType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; @@ -62,6 +64,11 @@ /** Test for {@link RowDataSerializer}. */ abstract class RowDataSerializerTest extends SerializerTestInstance { + private static final byte[] POINT_WKB = + new byte[] { + 1, GeographyData.POINT, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + private final RowDataSerializer serializer; private final RowData[] testData; @@ -233,6 +240,28 @@ private static RowDataSerializer getRowSerializer() { } } + static final class RowDataSerializerWithGeographyTest extends RowDataSerializerTest { + public RowDataSerializerWithGeographyTest() { + super(getRowSerializer(), getData()); + } + + private static RowData[] getData() { + GenericRowData row1 = new GenericRowData(1); + row1.setField(0, GeographyData.fromBytes(POINT_WKB)); + + GenericRowData row2 = new GenericRowData(1); + row2.setField(0, null); + + return new RowData[] {row1, row2}; + } + + private static RowDataSerializer getRowSerializer() { + InternalTypeInfo typeInfo = InternalTypeInfo.ofFields(new GeographyType()); + + return typeInfo.toRowSerializer(); + } + } + static final class LargeRowDataSerializerTest extends RowDataSerializerTest { public LargeRowDataSerializerTest() { super(getRowSerializer(), getData()); diff --git a/flink-tests/pom.xml b/flink-tests/pom.xml index 4ef3c5f273d53c..ee49aca935c6d1 100644 --- a/flink-tests/pom.xml +++ b/flink-tests/pom.xml @@ -69,6 +69,13 @@ under the License. test + + org.apache.flink + flink-connector-base + ${project.version} + test + + org.apache.flink flink-connector-files @@ -205,6 +212,13 @@ under the License. test + + org.apache.flink + flink-table-type-utils + ${project.version} + test + + org.apache.flink flink-runtime diff --git a/flink-tests/src/test/java/org/apache/flink/test/completeness/TypeSerializerTestCoverageTest.java b/flink-tests/src/test/java/org/apache/flink/test/completeness/TypeSerializerTestCoverageTest.java index 0ac026145139e6..84ada2f5b4271e 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/completeness/TypeSerializerTestCoverageTest.java +++ b/flink-tests/src/test/java/org/apache/flink/test/completeness/TypeSerializerTestCoverageTest.java @@ -83,6 +83,7 @@ import org.apache.flink.table.runtime.typeutils.BinaryRowDataSerializer; import org.apache.flink.table.runtime.typeutils.DecimalDataSerializer; import org.apache.flink.table.runtime.typeutils.ExternalSerializer; +import org.apache.flink.table.runtime.typeutils.GeographyTypeSerializer; import org.apache.flink.table.runtime.typeutils.LinkedListSerializer; import org.apache.flink.table.runtime.typeutils.MapDataSerializer; import org.apache.flink.table.runtime.typeutils.RawValueDataSerializer; @@ -196,6 +197,7 @@ void testTypeSerializerTestCoverage() { SharedBufferEdge.SharedBufferEdgeSerializer.class.getName(), RowDataSerializer.class.getName(), DecimalDataSerializer.class.getName(), + GeographyTypeSerializer.class.getName(), AvroSerializer.class.getName()); // type serializer whitelist for TypeSerializerUpgradeTestBase test coverage @@ -258,6 +260,7 @@ void testTypeSerializerTestCoverage() { SharedBufferEdge.SharedBufferEdgeSerializer.class.getName(), RowDataSerializer.class.getName(), DecimalDataSerializer.class.getName(), + GeographyTypeSerializer.class.getName(), AvroSerializer.class.getName(), // KeyAndValueSerializer shouldn't be used to serialize data to state and // doesn't need to ensure upgrade compatibility. diff --git a/pom.xml b/pom.xml index 670753e010bcb6..5e97251b027403 100644 --- a/pom.xml +++ b/pom.xml @@ -1741,6 +1741,8 @@ under the License. flink-python/docs/_build/** flink-python/docs/_static/switcher.json flink-python/docs/uv.lock + flink-python/pyflink/fn_execution/flink_fn_execution_pb2.py + flink-python/pyflink/fn_execution/flink_fn_execution_pb2.pyi **/awssdk/global/handlers/execution.interceptors