Skip to content

Commit 3430deb

Browse files
currantwcurrantw
authored
#3145 Add IP Address Data Type (#3175)
* Add `ExprIpValue` and `IP` data type Signed-off-by: currantw <[email protected]> * Add support for casting (`cast(field_name to ip)`) and remove existing unused sorting syntax. Signed-off-by: currantw <[email protected]> * Update comparison logic to compare in IPv6 Signed-off-by: currantw <[email protected]> * Fix bug casting to IP Signed-off-by: currantw <[email protected]> * Fix failing tests Signed-off-by: currantw <[email protected]> * Assert that comparison only valid if same type, update tests accordingly Signed-off-by: currantw <[email protected]> * Add additional tests to increase code coverage Signed-off-by: currantw <[email protected]> * Integrate `cidrmatch` changes Signed-off-by: currantw <[email protected]> * Remove `OpenSearchIPType` data type Signed-off-by: currantw <[email protected]> * Fix more failing tests Signed-off-by: currantw <[email protected]> * Minor cleanup Signed-off-by: currantw <[email protected]> * Add new tests for IP data type to `SortCommandIT`, and update `weblogs` test data. Signed-off-by: currantw <[email protected]> * Fixing IT test failure. Signed-off-by: currantw <[email protected]> * Spotless and update test to sort in SQL Signed-off-by: currantw <[email protected]> * Fix broken link Signed-off-by: currantw <[email protected]> * Fix failing code coverage Signed-off-by: currantw <[email protected]> * Fix failing doctest Signed-off-by: currantw <[email protected]> * Fix failing `ip.rst` doctest Signed-off-by: currantw <[email protected]> * Fix test failure due to merge. Signed-off-by: currantw <[email protected]> * Fix spotless Signed-off-by: currantw <[email protected]> * Add missing `url` field Signed-off-by: currantw <[email protected]> * Address minor review comments. Signed-off-by: currantw <[email protected]> * Revert sort syntax changes Signed-off-by: currantw <[email protected]> * Minor doc update Signed-off-by: currantw <[email protected]> * FIx failing `ip.rst` doctest Signed-off-by: currantw <[email protected]> * Add `IPComparisonIT` tests for comparison operators, rename modules and weblogs test index to make plural for consistency. Signed-off-by: currantw <[email protected]> --------- Signed-off-by: currantw <[email protected]>
1 parent 8bfa2e9 commit 3430deb

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+803
-400
lines changed

‎DEVELOPER_GUIDE.rst

+1-1
Original file line numberDiff line numberDiff line change
@@ -405,7 +405,7 @@ Sample test class:
405405
Doctest
406406
>>>>>>>
407407

408-
Python doctest library makes our document executable which keeps it up-to-date to source code. The doc generator aforementioned served as scaffolding and generated many docs in short time. Now the examples inside is changed to doctest gradually. For more details please read `Doctest <./dev/Doctest.md>`_.
408+
Python doctest library makes our document executable which keeps it up-to-date to source code. The doc generator aforementioned served as scaffolding and generated many docs in short time. Now the examples inside is changed to doctest gradually. For more details please read `testing-doctest <./docs/dev/testing-doctest.md>`_.
409409

410410

411411
Backports

‎core/src/main/java/org/opensearch/sql/ast/expression/Cast.java

+2
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_DOUBLE;
1313
import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_FLOAT;
1414
import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_INT;
15+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_IP;
1516
import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_LONG;
1617
import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_SHORT;
1718
import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_STRING;
@@ -54,6 +55,7 @@ public class Cast extends UnresolvedExpression {
5455
.put("time", CAST_TO_TIME.getName())
5556
.put("timestamp", CAST_TO_TIMESTAMP.getName())
5657
.put("datetime", CAST_TO_DATETIME.getName())
58+
.put("ip", CAST_TO_IP.getName())
5759
.build();
5860

5961
/** The source expression cast from. */

‎core/src/main/java/org/opensearch/sql/data/model/ExprIpValue.java

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.sql.data.model;
7+
8+
import inet.ipaddr.IPAddress;
9+
import org.opensearch.sql.data.type.ExprCoreType;
10+
import org.opensearch.sql.data.type.ExprType;
11+
import org.opensearch.sql.utils.IPUtils;
12+
13+
/** Expression IP Address Value. */
14+
public class ExprIpValue extends AbstractExprValue {
15+
private final IPAddress value;
16+
17+
public ExprIpValue(String addressString) {
18+
value = IPUtils.toAddress(addressString);
19+
}
20+
21+
@Override
22+
public String value() {
23+
return value.toCanonicalString();
24+
}
25+
26+
@Override
27+
public ExprType type() {
28+
return ExprCoreType.IP;
29+
}
30+
31+
@Override
32+
public int compare(ExprValue other) {
33+
return IPUtils.compare(value, ((ExprIpValue) other).value);
34+
}
35+
36+
@Override
37+
public boolean equal(ExprValue other) {
38+
return compare(other) == 0;
39+
}
40+
41+
@Override
42+
public String toString() {
43+
return String.format("IP %s", value());
44+
}
45+
46+
@Override
47+
public IPAddress ipValue() {
48+
return value;
49+
}
50+
}

‎core/src/main/java/org/opensearch/sql/data/model/ExprValue.java

+6
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
package org.opensearch.sql.data.model;
77

8+
import inet.ipaddr.IPAddress;
89
import java.io.Serializable;
910
import java.time.Instant;
1011
import java.time.LocalDate;
@@ -102,6 +103,11 @@ default Double doubleValue() {
102103
"invalid to get doubleValue from value of type " + type());
103104
}
104105

106+
/** Get IP address value. */
107+
default IPAddress ipValue() {
108+
throw new ExpressionEvaluationException("invalid to get ipValue from value of type " + type());
109+
}
110+
105111
/** Get string value. */
106112
default String stringValue() {
107113
throw new ExpressionEvaluationException(

‎core/src/main/java/org/opensearch/sql/data/model/ExprValueUtils.java

+9
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
package org.opensearch.sql.data.model;
77

8+
import inet.ipaddr.IPAddress;
89
import java.time.Instant;
910
import java.time.LocalDate;
1011
import java.time.LocalDateTime;
@@ -75,6 +76,10 @@ public static ExprValue timestampValue(Instant value) {
7576
return new ExprTimestampValue(value);
7677
}
7778

79+
public static ExprValue ipValue(String value) {
80+
return new ExprIpValue(value);
81+
}
82+
7883
/** {@link ExprTupleValue} constructor. */
7984
public static ExprValue tupleValue(Map<String, Object> map) {
8085
LinkedHashMap<String, ExprValue> valueMap = new LinkedHashMap<>();
@@ -188,6 +193,10 @@ public static Map<String, ExprValue> getTupleValue(ExprValue exprValue) {
188193
return exprValue.tupleValue();
189194
}
190195

196+
public static IPAddress getIpValue(ExprValue exprValue) {
197+
return exprValue.ipValue();
198+
}
199+
191200
public static Boolean getBooleanValue(ExprValue exprValue) {
192201
return exprValue.booleanValue();
193202
}

‎core/src/main/java/org/opensearch/sql/data/type/ExprCoreType.java

+3
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,9 @@ public enum ExprCoreType implements ExprType {
4545
TIMESTAMP(STRING, DATE, TIME),
4646
INTERVAL(UNDEFINED),
4747

48+
/** IP Address. */
49+
IP(STRING),
50+
4851
/** Struct. */
4952
STRUCT(UNDEFINED),
5053

‎core/src/main/java/org/opensearch/sql/expression/DSL.java

+4
Original file line numberDiff line numberDiff line change
@@ -835,6 +835,10 @@ public static FunctionExpression castTimestamp(Expression value) {
835835
return compile(FunctionProperties.None, BuiltinFunctionName.CAST_TO_TIMESTAMP, value);
836836
}
837837

838+
public static FunctionExpression castIp(Expression value) {
839+
return compile(FunctionProperties.None, BuiltinFunctionName.CAST_TO_IP, value);
840+
}
841+
838842
public static FunctionExpression typeof(Expression value) {
839843
return compile(FunctionProperties.None, BuiltinFunctionName.TYPEOF, value);
840844
}

‎core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java

+1
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,7 @@ public enum BuiltinFunctionName {
231231
CAST_TO_TIME(FunctionName.of("cast_to_time")),
232232
CAST_TO_TIMESTAMP(FunctionName.of("cast_to_timestamp")),
233233
CAST_TO_DATETIME(FunctionName.of("cast_to_datetime")),
234+
CAST_TO_IP(FunctionName.of("cast_to_ip")),
234235
TYPEOF(FunctionName.of("typeof")),
235236

236237
/** Relevance Function. */

‎core/src/main/java/org/opensearch/sql/expression/ip/IPFunctions.java

+12-57
Original file line numberDiff line numberDiff line change
@@ -6,21 +6,21 @@
66
package org.opensearch.sql.expression.ip;
77

88
import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN;
9+
import static org.opensearch.sql.data.type.ExprCoreType.IP;
910
import static org.opensearch.sql.data.type.ExprCoreType.STRING;
1011
import static org.opensearch.sql.expression.function.FunctionDSL.define;
1112
import static org.opensearch.sql.expression.function.FunctionDSL.impl;
1213
import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling;
1314

14-
import inet.ipaddr.AddressStringException;
15-
import inet.ipaddr.IPAddressString;
16-
import inet.ipaddr.IPAddressStringParameters;
15+
import inet.ipaddr.IPAddress;
1716
import lombok.experimental.UtilityClass;
1817
import org.opensearch.sql.data.model.ExprValue;
1918
import org.opensearch.sql.data.model.ExprValueUtils;
2019
import org.opensearch.sql.exception.SemanticCheckException;
2120
import org.opensearch.sql.expression.function.BuiltinFunctionName;
2221
import org.opensearch.sql.expression.function.BuiltinFunctionRepository;
2322
import org.opensearch.sql.expression.function.DefaultFunctionResolver;
23+
import org.opensearch.sql.utils.IPUtils;
2424

2525
/** Utility class that defines and registers IP functions. */
2626
@UtilityClass
@@ -31,75 +31,30 @@ public void register(BuiltinFunctionRepository repository) {
3131
}
3232

3333
private DefaultFunctionResolver cidrmatch() {
34-
35-
// TODO #3145: Add support for IP address data type.
3634
return define(
3735
BuiltinFunctionName.CIDRMATCH.getName(),
38-
impl(nullMissingHandling(IPFunctions::exprCidrMatch), BOOLEAN, STRING, STRING));
36+
impl(nullMissingHandling(IPFunctions::exprCidrMatch), BOOLEAN, IP, STRING));
3937
}
4038

4139
/**
4240
* Returns whether the given IP address is within the specified inclusive CIDR IP address range.
4341
* Supports both IPv4 and IPv6 addresses.
4442
*
45-
* @param addressExprValue IP address as a string (e.g. "198.51.100.14" or
46-
* "2001:0db8::ff00:42:8329").
47-
* @param rangeExprValue IP address range in CIDR notation as a string (e.g. "198.51.100.0/24" or
43+
* @param addressExprValue IP address (e.g. "198.51.100.14" or "2001:0db8::ff00:42:8329").
44+
* @param rangeExprValue IP address range string in CIDR notation (e.g. "198.51.100.0/24" or
4845
* "2001:0db8::/32")
4946
* @return true if the address is in the range; otherwise false.
5047
* @throws SemanticCheckException if the address or range is not valid, or if they do not use the
5148
* same version (IPv4 or IPv6).
5249
*/
5350
private ExprValue exprCidrMatch(ExprValue addressExprValue, ExprValue rangeExprValue) {
5451

55-
// TODO #3145: Update to support IP address data type.
56-
String addressString = addressExprValue.stringValue();
57-
String rangeString = rangeExprValue.stringValue();
58-
59-
final IPAddressStringParameters validationOptions =
60-
new IPAddressStringParameters.Builder()
61-
.allowEmpty(false)
62-
.setEmptyAsLoopback(false)
63-
.allow_inet_aton(false)
64-
.allowSingleSegment(false)
65-
.toParams();
66-
67-
// Get and validate IP address.
68-
IPAddressString address =
69-
new IPAddressString(addressExprValue.stringValue(), validationOptions);
70-
71-
try {
72-
address.validate();
73-
} catch (AddressStringException e) {
74-
String msg =
75-
String.format(
76-
"IP address '%s' is not valid. Error details: %s", addressString, e.getMessage());
77-
throw new SemanticCheckException(msg, e);
78-
}
79-
80-
// Get and validate CIDR IP address range.
81-
IPAddressString range = new IPAddressString(rangeExprValue.stringValue(), validationOptions);
82-
83-
try {
84-
range.validate();
85-
} catch (AddressStringException e) {
86-
String msg =
87-
String.format(
88-
"CIDR IP address range '%s' is not valid. Error details: %s",
89-
rangeString, e.getMessage());
90-
throw new SemanticCheckException(msg, e);
91-
}
92-
93-
// Address and range must use the same IP version (IPv4 or IPv6).
94-
if (address.isIPv4() ^ range.isIPv4()) {
95-
String msg =
96-
String.format(
97-
"IP address '%s' and CIDR IP address range '%s' are not compatible. Both must be"
98-
+ " either IPv4 or IPv6.",
99-
addressString, rangeString);
100-
throw new SemanticCheckException(msg);
101-
}
52+
IPAddress address = addressExprValue.ipValue();
53+
IPAddress range = IPUtils.toRange(rangeExprValue.stringValue());
10254

103-
return ExprValueUtils.booleanValue(range.contains(address));
55+
return (IPUtils.compare(address, range.getLower()) < 0)
56+
|| (IPUtils.compare(address, range.getUpper()) > 0)
57+
? ExprValueUtils.LITERAL_FALSE
58+
: ExprValueUtils.LITERAL_TRUE;
10459
}
10560
}

‎core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java

+10
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE;
1212
import static org.opensearch.sql.data.type.ExprCoreType.FLOAT;
1313
import static org.opensearch.sql.data.type.ExprCoreType.INTEGER;
14+
import static org.opensearch.sql.data.type.ExprCoreType.IP;
1415
import static org.opensearch.sql.data.type.ExprCoreType.LONG;
1516
import static org.opensearch.sql.data.type.ExprCoreType.SHORT;
1617
import static org.opensearch.sql.data.type.ExprCoreType.STRING;
@@ -31,6 +32,7 @@
3132
import org.opensearch.sql.data.model.ExprDoubleValue;
3233
import org.opensearch.sql.data.model.ExprFloatValue;
3334
import org.opensearch.sql.data.model.ExprIntegerValue;
35+
import org.opensearch.sql.data.model.ExprIpValue;
3436
import org.opensearch.sql.data.model.ExprLongValue;
3537
import org.opensearch.sql.data.model.ExprShortValue;
3638
import org.opensearch.sql.data.model.ExprStringValue;
@@ -54,6 +56,7 @@ public static void register(BuiltinFunctionRepository repository) {
5456
repository.register(castToFloat());
5557
repository.register(castToDouble());
5658
repository.register(castToBoolean());
59+
repository.register(castToIp());
5760
repository.register(castToDate());
5861
repository.register(castToTime());
5962
repository.register(castToTimestamp());
@@ -173,6 +176,13 @@ private static DefaultFunctionResolver castToBoolean() {
173176
impl(nullMissingHandling((v) -> v), BOOLEAN, BOOLEAN));
174177
}
175178

179+
private static DefaultFunctionResolver castToIp() {
180+
return FunctionDSL.define(
181+
BuiltinFunctionName.CAST_TO_IP.getName(),
182+
impl(nullMissingHandling((v) -> new ExprIpValue(v.stringValue())), IP, STRING),
183+
impl(nullMissingHandling((v) -> v), IP, IP));
184+
}
185+
176186
private static DefaultFunctionResolver castToDate() {
177187
return FunctionDSL.define(
178188
BuiltinFunctionName.CAST_TO_DATE.getName(),

0 commit comments

Comments
 (0)