spark-rapids
spark-rapids copied to clipboard
[BUG] DecimalGen does not produce negative values or MIN/MAX/0 values as special cases
Describe the bug This reduces the quality of the tests.
I have a patch for this.
diff --git a/integration_tests/src/main/python/data_gen.py b/integration_tests/src/main/python/data_gen.py
index ac3c812a5..4f864691a 100644
--- a/integration_tests/src/main/python/data_gen.py
+++ b/integration_tests/src/main/python/data_gen.py
@@ -215,17 +215,19 @@ class IntegerGen(DataGen):
class DecimalGen(DataGen):
"""Generate Decimals, with some built in corner cases."""
- def __init__(self, precision=None, scale=None, nullable=True, special_cases=[]):
+ def __init__(self, precision=None, scale=None, nullable=True, special_cases=None):
if precision is None:
#Maximum number of decimal digits a Long can represent is 18
precision = 18
scale = 0
DECIMAL_MIN = Decimal('-' + ('9' * precision) + 'e' + str(-scale))
DECIMAL_MAX = Decimal(('9'* precision) + 'e' + str(-scale))
+ if (special_cases is None):
+ special_cases = [DECIMAL_MIN, DECIMAL_MAX, Decimal('0')]
super().__init__(DecimalType(precision, scale), nullable=nullable, special_cases=special_cases)
self.scale = scale
self.precision = precision
- pattern = "[0-9]{1,"+ str(precision) + "}e" + str(-scale)
+ pattern = "-?[0-9]{1,"+ str(precision) + "}e" + str(-scale)
self.base_strs = sre_yield.AllStrings(pattern, flags=0, charset=sre_yield.CHARSET, max_count=_MAX_CHOICES)
def __repr__(self):
But I have run into a number of different issues. In our code and in Spark
https://issues.apache.org/jira/browse/SPARK-40089 https://github.com/NVIDIA/spark-rapids/issues/6336
and I am working on filing a few more issues.
Steps/Code to reproduce bug apply the patch and run...