From 7f2042742bac968e249aeac2f18194723fb247b6 Mon Sep 17 00:00:00 2001 From: Colin Dellow Date: Sun, 18 Mar 2018 17:49:12 -0400 Subject: [PATCH] Also compare queries against SQLite itself --- .gitignore | 2 + parquet-generator/no-nulls.sql | 102 ++++++++++++++++++++++++++++ parquet-generator/nulls.sql | 102 ++++++++++++++++++++++++++++ parquet-generator/parquets.py | 88 ++++++++++++++++++++++-- tests/create-queries-from-templates | 4 +- tests/test-queries | 19 +++--- 6 files changed, 300 insertions(+), 17 deletions(-) create mode 100644 parquet-generator/no-nulls.sql create mode 100644 parquet-generator/nulls.sql diff --git a/.gitignore b/.gitignore index 67af04b..f22cd63 100644 --- a/.gitignore +++ b/.gitignore @@ -46,3 +46,5 @@ /testcase-cmds.txt /testcases.txt tests/queries +/testcase-bootstrap.sql +/test.db diff --git a/parquet-generator/no-nulls.sql b/parquet-generator/no-nulls.sql new file mode 100644 index 0000000..e1a5362 --- /dev/null +++ b/parquet-generator/no-nulls.sql @@ -0,0 +1,102 @@ +BEGIN;DROP TABLE IF EXISTS no_nulls; +CREATE TABLE no_nulls (bool_0 BOOLEAN, int8_1 TINYINT, int16_2 SMALLINT, int32_3 INT, int64_4 BIGINT, ts_5 BIGINT, double_6 DOUBLE, string_7 TEXT, string_8 TEXT, binary_9 BLOB, binary_10 BLOB); +INSERT INTO no_nulls VALUES (1, 50, 5000, 50000000, 50000000000, 490665600000, 99.0, '0', '000', X'00', X'00'); +INSERT INTO no_nulls VALUES (0, 49, 4900, 49000000, 49000000000, 490752000000, 49.5, '1', '001', X'0101', X'01'); +INSERT INTO no_nulls VALUES (1, 48, 4800, 48000000, 48000000000, 490838400000, 33.0, '2', '002', X'020202', X'02'); +INSERT INTO no_nulls VALUES (0, 47, 4700, 47000000, 47000000000, 490924800000, 24.75, '3', '003', X'03030303', X'03'); +INSERT INTO no_nulls VALUES (1, 46, 4600, 46000000, 46000000000, 491011200000, 19.8, '4', '004', X'0404040404', X'04'); +INSERT INTO no_nulls VALUES (0, 45, 4500, 45000000, 45000000000, 491097600000, 16.5, '5', '005', X'05', X'05'); +INSERT INTO no_nulls VALUES (1, 44, 4400, 44000000, 44000000000, 491184000000, 14.142857142857142, '6', '006', X'0606', X'06'); +INSERT INTO no_nulls VALUES (0, 43, 4300, 43000000, 43000000000, 491270400000, 12.375, '7', '007', X'070707', X'07'); +INSERT INTO no_nulls VALUES (1, 42, 4200, 42000000, 42000000000, 491356800000, 11.0, '8', '008', X'08080808', X'08'); +INSERT INTO no_nulls VALUES (0, 41, 4100, 41000000, 41000000000, 491443200000, 9.9, '9', '009', X'0909090909', X'09'); +INSERT INTO no_nulls VALUES (1, 40, 4000, 40000000, 40000000000, 491529600000, 9.0, '10', '010', X'0A', X'0A'); +INSERT INTO no_nulls VALUES (0, 39, 3900, 39000000, 39000000000, 491616000000, 8.25, '11', '011', X'0B0B', X'0B'); +INSERT INTO no_nulls VALUES (1, 38, 3800, 38000000, 38000000000, 491702400000, 7.615384615384615, '12', '012', X'0C0C0C', X'0C'); +INSERT INTO no_nulls VALUES (0, 37, 3700, 37000000, 37000000000, 491788800000, 7.071428571428571, '13', '013', X'0D0D0D0D', X'0D'); +INSERT INTO no_nulls VALUES (1, 36, 3600, 36000000, 36000000000, 491875200000, 6.6, '14', '014', X'0E0E0E0E0E', X'0E'); +INSERT INTO no_nulls VALUES (0, 35, 3500, 35000000, 35000000000, 491961600000, 6.1875, '15', '015', X'0F', X'0F'); +INSERT INTO no_nulls VALUES (1, 34, 3400, 34000000, 34000000000, 492048000000, 5.823529411764706, '16', '016', X'1010', X'10'); +INSERT INTO no_nulls VALUES (0, 33, 3300, 33000000, 33000000000, 492134400000, 5.5, '17', '017', X'111111', X'11'); +INSERT INTO no_nulls VALUES (1, 32, 3200, 32000000, 32000000000, 492220800000, 5.2105263157894735, '18', '018', X'12121212', X'12'); +INSERT INTO no_nulls VALUES (0, 31, 3100, 31000000, 31000000000, 492307200000, 4.95, '19', '019', X'1313131313', X'13'); +INSERT INTO no_nulls VALUES (1, 30, 3000, 30000000, 30000000000, 492393600000, 4.714285714285714, '20', '020', X'14', X'14'); +INSERT INTO no_nulls VALUES (0, 29, 2900, 29000000, 29000000000, 492480000000, 4.5, '21', '021', X'1515', X'15'); +INSERT INTO no_nulls VALUES (1, 28, 2800, 28000000, 28000000000, 492566400000, 4.304347826086956, '22', '022', X'161616', X'16'); +INSERT INTO no_nulls VALUES (0, 27, 2700, 27000000, 27000000000, 492652800000, 4.125, '23', '023', X'17171717', X'17'); +INSERT INTO no_nulls VALUES (1, 26, 2600, 26000000, 26000000000, 492739200000, 3.96, '24', '024', X'1818181818', X'18'); +INSERT INTO no_nulls VALUES (0, 25, 2500, 25000000, 25000000000, 492825600000, 3.8076923076923075, '25', '025', X'19', X'19'); +INSERT INTO no_nulls VALUES (1, 24, 2400, 24000000, 24000000000, 492912000000, 3.6666666666666665, '26', '026', X'1A1A', X'1A'); +INSERT INTO no_nulls VALUES (0, 23, 2300, 23000000, 23000000000, 492998400000, 3.5357142857142856, '27', '027', X'1B1B1B', X'1B'); +INSERT INTO no_nulls VALUES (1, 22, 2200, 22000000, 22000000000, 493084800000, 3.413793103448276, '28', '028', X'1C1C1C1C', X'1C'); +INSERT INTO no_nulls VALUES (0, 21, 2100, 21000000, 21000000000, 493171200000, 3.3, '29', '029', X'1D1D1D1D1D', X'1D'); +INSERT INTO no_nulls VALUES (1, 20, 2000, 20000000, 20000000000, 493257600000, 3.193548387096774, '30', '030', X'1E', X'1E'); +INSERT INTO no_nulls VALUES (0, 19, 1900, 19000000, 19000000000, 493344000000, 3.09375, '31', '031', X'1F1F', X'1F'); +INSERT INTO no_nulls VALUES (1, 18, 1800, 18000000, 18000000000, 493430400000, 3.0, '32', '032', X'202020', X'20'); +INSERT INTO no_nulls VALUES (0, 17, 1700, 17000000, 17000000000, 493516800000, 2.911764705882353, '33', '033', X'21212121', X'21'); +INSERT INTO no_nulls VALUES (1, 16, 1600, 16000000, 16000000000, 493603200000, 2.8285714285714287, '34', '034', X'2222222222', X'22'); +INSERT INTO no_nulls VALUES (0, 15, 1500, 15000000, 15000000000, 493689600000, 2.75, '35', '035', X'23', X'23'); +INSERT INTO no_nulls VALUES (1, 14, 1400, 14000000, 14000000000, 493776000000, 2.675675675675676, '36', '036', X'2424', X'24'); +INSERT INTO no_nulls VALUES (0, 13, 1300, 13000000, 13000000000, 493862400000, 2.6052631578947367, '37', '037', X'252525', X'25'); +INSERT INTO no_nulls VALUES (1, 12, 1200, 12000000, 12000000000, 493948800000, 2.5384615384615383, '38', '038', X'26262626', X'26'); +INSERT INTO no_nulls VALUES (0, 11, 1100, 11000000, 11000000000, 494035200000, 2.475, '39', '039', X'2727272727', X'27'); +INSERT INTO no_nulls VALUES (1, 10, 1000, 10000000, 10000000000, 494121600000, 2.4146341463414633, '40', '040', X'28', X'28'); +INSERT INTO no_nulls VALUES (0, 9, 900, 9000000, 9000000000, 494208000000, 2.357142857142857, '41', '041', X'2929', X'29'); +INSERT INTO no_nulls VALUES (1, 8, 800, 8000000, 8000000000, 494294400000, 2.302325581395349, '42', '042', X'2A2A2A', X'2A'); +INSERT INTO no_nulls VALUES (0, 7, 700, 7000000, 7000000000, 494380800000, 2.25, '43', '043', X'2B2B2B2B', X'2B'); +INSERT INTO no_nulls VALUES (1, 6, 600, 6000000, 6000000000, 494467200000, 2.2, '44', '044', X'2C2C2C2C2C', X'2C'); +INSERT INTO no_nulls VALUES (0, 5, 500, 5000000, 5000000000, 494553600000, 2.152173913043478, '45', '045', X'2D', X'2D'); +INSERT INTO no_nulls VALUES (1, 4, 400, 4000000, 4000000000, 494640000000, 2.106382978723404, '46', '046', X'2E2E', X'2E'); +INSERT INTO no_nulls VALUES (0, 3, 300, 3000000, 3000000000, 494726400000, 2.0625, '47', '047', X'2F2F2F', X'2F'); +INSERT INTO no_nulls VALUES (1, 2, 200, 2000000, 2000000000, 494812800000, 2.020408163265306, '48', '048', X'30303030', X'30'); +INSERT INTO no_nulls VALUES (0, 1, 100, 1000000, 1000000000, 494899200000, 1.98, '49', '049', X'3131313131', X'31'); +INSERT INTO no_nulls VALUES (1, 0, 0, 0, 0, 494985600000, 1.9411764705882353, '50', '050', X'32', X'32'); +INSERT INTO no_nulls VALUES (0, -1, -100, -1000000, -1000000000, 495072000000, 1.9038461538461537, '51', '051', X'3333', X'33'); +INSERT INTO no_nulls VALUES (1, -2, -200, -2000000, -2000000000, 495158400000, 1.8679245283018868, '52', '052', X'343434', X'34'); +INSERT INTO no_nulls VALUES (0, -3, -300, -3000000, -3000000000, 495244800000, 1.8333333333333333, '53', '053', X'35353535', X'35'); +INSERT INTO no_nulls VALUES (1, -4, -400, -4000000, -4000000000, 495331200000, 1.8, '54', '054', X'3636363636', X'36'); +INSERT INTO no_nulls VALUES (0, -5, -500, -5000000, -5000000000, 495417600000, 1.7678571428571428, '55', '055', X'37', X'37'); +INSERT INTO no_nulls VALUES (1, -6, -600, -6000000, -6000000000, 495504000000, 1.736842105263158, '56', '056', X'3838', X'38'); +INSERT INTO no_nulls VALUES (0, -7, -700, -7000000, -7000000000, 495590400000, 1.706896551724138, '57', '057', X'393939', X'39'); +INSERT INTO no_nulls VALUES (1, -8, -800, -8000000, -8000000000, 495676800000, 1.6779661016949152, '58', '058', X'3A3A3A3A', X'3A'); +INSERT INTO no_nulls VALUES (0, -9, -900, -9000000, -9000000000, 495763200000, 1.65, '59', '059', X'3B3B3B3B3B', X'3B'); +INSERT INTO no_nulls VALUES (1, -10, -1000, -10000000, -10000000000, 495849600000, 1.6229508196721312, '60', '060', X'3C', X'3C'); +INSERT INTO no_nulls VALUES (0, -11, -1100, -11000000, -11000000000, 495936000000, 1.596774193548387, '61', '061', X'3D3D', X'3D'); +INSERT INTO no_nulls VALUES (1, -12, -1200, -12000000, -12000000000, 496022400000, 1.5714285714285714, '62', '062', X'3E3E3E', X'3E'); +INSERT INTO no_nulls VALUES (0, -13, -1300, -13000000, -13000000000, 496108800000, 1.546875, '63', '063', X'3F3F3F3F', X'3F'); +INSERT INTO no_nulls VALUES (1, -14, -1400, -14000000, -14000000000, 496195200000, 1.523076923076923, '64', '064', X'4040404040', X'40'); +INSERT INTO no_nulls VALUES (0, -15, -1500, -15000000, -15000000000, 496281600000, 1.5, '65', '065', X'41', X'41'); +INSERT INTO no_nulls VALUES (1, -16, -1600, -16000000, -16000000000, 496368000000, 1.4776119402985075, '66', '066', X'4242', X'42'); +INSERT INTO no_nulls VALUES (0, -17, -1700, -17000000, -17000000000, 496454400000, 1.4558823529411764, '67', '067', X'434343', X'43'); +INSERT INTO no_nulls VALUES (1, -18, -1800, -18000000, -18000000000, 496540800000, 1.434782608695652, '68', '068', X'44444444', X'44'); +INSERT INTO no_nulls VALUES (0, -19, -1900, -19000000, -19000000000, 496627200000, 1.4142857142857144, '69', '069', X'4545454545', X'45'); +INSERT INTO no_nulls VALUES (1, -20, -2000, -20000000, -20000000000, 496713600000, 1.3943661971830985, '70', '070', X'46', X'46'); +INSERT INTO no_nulls VALUES (0, -21, -2100, -21000000, -21000000000, 496800000000, 1.375, '71', '071', X'4747', X'47'); +INSERT INTO no_nulls VALUES (1, -22, -2200, -22000000, -22000000000, 496886400000, 1.356164383561644, '72', '072', X'484848', X'48'); +INSERT INTO no_nulls VALUES (0, -23, -2300, -23000000, -23000000000, 496972800000, 1.337837837837838, '73', '073', X'49494949', X'49'); +INSERT INTO no_nulls VALUES (1, -24, -2400, -24000000, -24000000000, 497059200000, 1.32, '74', '074', X'4A4A4A4A4A', X'4A'); +INSERT INTO no_nulls VALUES (0, -25, -2500, -25000000, -25000000000, 497145600000, 1.3026315789473684, '75', '075', X'4B', X'4B'); +INSERT INTO no_nulls VALUES (1, -26, -2600, -26000000, -26000000000, 497232000000, 1.2857142857142858, '76', '076', X'4C4C', X'4C'); +INSERT INTO no_nulls VALUES (0, -27, -2700, -27000000, -27000000000, 497318400000, 1.2692307692307692, '77', '077', X'4D4D4D', X'4D'); +INSERT INTO no_nulls VALUES (1, -28, -2800, -28000000, -28000000000, 497404800000, 1.2531645569620253, '78', '078', X'4E4E4E4E', X'4E'); +INSERT INTO no_nulls VALUES (0, -29, -2900, -29000000, -29000000000, 497491200000, 1.2375, '79', '079', X'4F4F4F4F4F', X'4F'); +INSERT INTO no_nulls VALUES (1, -30, -3000, -30000000, -30000000000, 497577600000, 1.2222222222222223, '80', '080', X'50', X'50'); +INSERT INTO no_nulls VALUES (0, -31, -3100, -31000000, -31000000000, 497664000000, 1.2073170731707317, '81', '081', X'5151', X'51'); +INSERT INTO no_nulls VALUES (1, -32, -3200, -32000000, -32000000000, 497750400000, 1.1927710843373494, '82', '082', X'525252', X'52'); +INSERT INTO no_nulls VALUES (0, -33, -3300, -33000000, -33000000000, 497836800000, 1.1785714285714286, '83', '083', X'53535353', X'53'); +INSERT INTO no_nulls VALUES (1, -34, -3400, -34000000, -34000000000, 497923200000, 1.1647058823529413, '84', '084', X'5454545454', X'54'); +INSERT INTO no_nulls VALUES (0, -35, -3500, -35000000, -35000000000, 498009600000, 1.1511627906976745, '85', '085', X'55', X'55'); +INSERT INTO no_nulls VALUES (1, -36, -3600, -36000000, -36000000000, 498096000000, 1.1379310344827587, '86', '086', X'5656', X'56'); +INSERT INTO no_nulls VALUES (0, -37, -3700, -37000000, -37000000000, 498182400000, 1.125, '87', '087', X'575757', X'57'); +INSERT INTO no_nulls VALUES (1, -38, -3800, -38000000, -38000000000, 498268800000, 1.1123595505617978, '88', '088', X'58585858', X'58'); +INSERT INTO no_nulls VALUES (0, -39, -3900, -39000000, -39000000000, 498355200000, 1.1, '89', '089', X'5959595959', X'59'); +INSERT INTO no_nulls VALUES (1, -40, -4000, -40000000, -40000000000, 498441600000, 1.0879120879120878, '90', '090', X'5A', X'5A'); +INSERT INTO no_nulls VALUES (0, -41, -4100, -41000000, -41000000000, 498528000000, 1.076086956521739, '91', '091', X'5B5B', X'5B'); +INSERT INTO no_nulls VALUES (1, -42, -4200, -42000000, -42000000000, 498614400000, 1.064516129032258, '92', '092', X'5C5C5C', X'5C'); +INSERT INTO no_nulls VALUES (0, -43, -4300, -43000000, -43000000000, 498700800000, 1.053191489361702, '93', '093', X'5D5D5D5D', X'5D'); +INSERT INTO no_nulls VALUES (1, -44, -4400, -44000000, -44000000000, 498787200000, 1.0421052631578946, '94', '094', X'5E5E5E5E5E', X'5E'); +INSERT INTO no_nulls VALUES (0, -45, -4500, -45000000, -45000000000, 498873600000, 1.03125, '95', '095', X'5F', X'5F'); +INSERT INTO no_nulls VALUES (1, -46, -4600, -46000000, -46000000000, 498960000000, 1.0206185567010309, '96', '096', X'6060', X'60'); +INSERT INTO no_nulls VALUES (0, -47, -4700, -47000000, -47000000000, 499046400000, 1.010204081632653, '97', '097', X'616161', X'61'); +INSERT INTO no_nulls VALUES (1, -48, -4800, -48000000, -48000000000, 499132800000, 1, '98', '098', X'62626262', X'62'); +COMMIT; diff --git a/parquet-generator/nulls.sql b/parquet-generator/nulls.sql new file mode 100644 index 0000000..24b7728 --- /dev/null +++ b/parquet-generator/nulls.sql @@ -0,0 +1,102 @@ +BEGIN;DROP TABLE IF EXISTS nulls; +CREATE TABLE nulls (bool_0 BOOLEAN, int8_1 TINYINT, int16_2 SMALLINT, int32_3 INT, int64_4 BIGINT, ts_5 BIGINT, double_6 DOUBLE, string_7 TEXT, string_8 TEXT, binary_9 BLOB, binary_10 BLOB); +INSERT INTO nulls VALUES (1, 50, 5000, 50000000, 50000000000, 490665600000, 99.0, '0', '000', X'00', X'00'); +INSERT INTO nulls VALUES (0, 49, 4900, 49000000, 49000000000, 490752000000, 49.5, '1', '001', X'0101', X'01'); +INSERT INTO nulls VALUES (1, 48, 4800, 48000000, 48000000000, 490838400000, 33.0, '2', '002', X'020202', X'02'); +INSERT INTO nulls VALUES (0, 47, 4700, 47000000, 47000000000, 490924800000, 24.75, '3', '003', X'03030303', X'03'); +INSERT INTO nulls VALUES (1, 46, 4600, 46000000, 46000000000, 491011200000, 19.8, '4', '004', X'0404040404', X'04'); +INSERT INTO nulls VALUES (0, 45, 4500, 45000000, 45000000000, 491097600000, 16.5, '5', '005', X'05', X'05'); +INSERT INTO nulls VALUES (1, 44, 4400, 44000000, 44000000000, 491184000000, 14.142857142857142, '6', '006', X'0606', X'06'); +INSERT INTO nulls VALUES (0, 43, 4300, 43000000, 43000000000, 491270400000, 12.375, '7', '007', X'070707', X'07'); +INSERT INTO nulls VALUES (1, 42, 4200, 42000000, 42000000000, 491356800000, 11.0, '8', '008', X'08080808', X'08'); +INSERT INTO nulls VALUES (0, 41, 4100, 41000000, 41000000000, 491443200000, 9.9, '9', '009', X'0909090909', X'09'); +INSERT INTO nulls VALUES (NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); +INSERT INTO nulls VALUES (NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); +INSERT INTO nulls VALUES (NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); +INSERT INTO nulls VALUES (NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); +INSERT INTO nulls VALUES (NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); +INSERT INTO nulls VALUES (NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); +INSERT INTO nulls VALUES (NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); +INSERT INTO nulls VALUES (NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); +INSERT INTO nulls VALUES (NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); +INSERT INTO nulls VALUES (NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); +INSERT INTO nulls VALUES (NULL, 30, NULL, 30000000, NULL, 492393600000, NULL, '20', NULL, X'14', NULL); +INSERT INTO nulls VALUES (0, NULL, 2900, NULL, 29000000000, NULL, 4.5, NULL, '021', NULL, X'15'); +INSERT INTO nulls VALUES (NULL, 28, NULL, 28000000, NULL, 492566400000, NULL, '22', NULL, X'161616', NULL); +INSERT INTO nulls VALUES (0, NULL, 2700, NULL, 27000000000, NULL, 4.125, NULL, '023', NULL, X'17'); +INSERT INTO nulls VALUES (NULL, 26, NULL, 26000000, NULL, 492739200000, NULL, '24', NULL, X'1818181818', NULL); +INSERT INTO nulls VALUES (0, NULL, 2500, NULL, 25000000000, NULL, 3.8076923076923075, NULL, '025', NULL, X'19'); +INSERT INTO nulls VALUES (NULL, 24, NULL, 24000000, NULL, 492912000000, NULL, '26', NULL, X'1A1A', NULL); +INSERT INTO nulls VALUES (0, NULL, 2300, NULL, 23000000000, NULL, 3.5357142857142856, NULL, '027', NULL, X'1B'); +INSERT INTO nulls VALUES (NULL, 22, NULL, 22000000, NULL, 493084800000, NULL, '28', NULL, X'1C1C1C1C', NULL); +INSERT INTO nulls VALUES (0, NULL, 2100, NULL, 21000000000, NULL, 3.3, NULL, '029', NULL, X'1D'); +INSERT INTO nulls VALUES (NULL, 20, NULL, 20000000, NULL, 493257600000, NULL, '30', NULL, X'1E', NULL); +INSERT INTO nulls VALUES (0, NULL, 1900, NULL, 19000000000, NULL, 3.09375, NULL, '031', NULL, X'1F'); +INSERT INTO nulls VALUES (NULL, 18, NULL, 18000000, NULL, 493430400000, NULL, '32', NULL, X'202020', NULL); +INSERT INTO nulls VALUES (0, NULL, 1700, NULL, 17000000000, NULL, 2.911764705882353, NULL, '033', NULL, X'21'); +INSERT INTO nulls VALUES (NULL, 16, NULL, 16000000, NULL, 493603200000, NULL, '34', NULL, X'2222222222', NULL); +INSERT INTO nulls VALUES (0, NULL, 1500, NULL, 15000000000, NULL, 2.75, NULL, '035', NULL, X'23'); +INSERT INTO nulls VALUES (NULL, 14, NULL, 14000000, NULL, 493776000000, NULL, '36', NULL, X'2424', NULL); +INSERT INTO nulls VALUES (0, NULL, 1300, NULL, 13000000000, NULL, 2.6052631578947367, NULL, '037', NULL, X'25'); +INSERT INTO nulls VALUES (NULL, 12, NULL, 12000000, NULL, 493948800000, NULL, '38', NULL, X'26262626', NULL); +INSERT INTO nulls VALUES (0, NULL, 1100, NULL, 11000000000, NULL, 2.475, NULL, '039', NULL, X'27'); +INSERT INTO nulls VALUES (NULL, 10, NULL, 10000000, NULL, 494121600000, NULL, '40', NULL, X'28', NULL); +INSERT INTO nulls VALUES (0, NULL, 900, NULL, 9000000000, NULL, 2.357142857142857, NULL, '041', NULL, X'29'); +INSERT INTO nulls VALUES (NULL, 8, NULL, 8000000, NULL, 494294400000, NULL, '42', NULL, X'2A2A2A', NULL); +INSERT INTO nulls VALUES (0, NULL, 700, NULL, 7000000000, NULL, 2.25, NULL, '043', NULL, X'2B'); +INSERT INTO nulls VALUES (NULL, 6, NULL, 6000000, NULL, 494467200000, NULL, '44', NULL, X'2C2C2C2C2C', NULL); +INSERT INTO nulls VALUES (0, NULL, 500, NULL, 5000000000, NULL, 2.152173913043478, NULL, '045', NULL, X'2D'); +INSERT INTO nulls VALUES (NULL, 4, NULL, 4000000, NULL, 494640000000, NULL, '46', NULL, X'2E2E', NULL); +INSERT INTO nulls VALUES (0, NULL, 300, NULL, 3000000000, NULL, 2.0625, NULL, '047', NULL, X'2F'); +INSERT INTO nulls VALUES (NULL, 2, NULL, 2000000, NULL, 494812800000, NULL, '48', NULL, X'30303030', NULL); +INSERT INTO nulls VALUES (0, NULL, 100, NULL, 1000000000, NULL, 1.98, NULL, '049', NULL, X'31'); +INSERT INTO nulls VALUES (NULL, 0, NULL, 0, NULL, 494985600000, NULL, '50', NULL, X'32', NULL); +INSERT INTO nulls VALUES (0, NULL, -100, NULL, -1000000000, NULL, 1.9038461538461537, NULL, '051', NULL, X'33'); +INSERT INTO nulls VALUES (NULL, -2, NULL, -2000000, NULL, 495158400000, NULL, '52', NULL, X'343434', NULL); +INSERT INTO nulls VALUES (0, NULL, -300, NULL, -3000000000, NULL, 1.8333333333333333, NULL, '053', NULL, X'35'); +INSERT INTO nulls VALUES (NULL, -4, NULL, -4000000, NULL, 495331200000, NULL, '54', NULL, X'3636363636', NULL); +INSERT INTO nulls VALUES (0, NULL, -500, NULL, -5000000000, NULL, 1.7678571428571428, NULL, '055', NULL, X'37'); +INSERT INTO nulls VALUES (NULL, -6, NULL, -6000000, NULL, 495504000000, NULL, '56', NULL, X'3838', NULL); +INSERT INTO nulls VALUES (0, NULL, -700, NULL, -7000000000, NULL, 1.706896551724138, NULL, '057', NULL, X'39'); +INSERT INTO nulls VALUES (NULL, -8, NULL, -8000000, NULL, 495676800000, NULL, '58', NULL, X'3A3A3A3A', NULL); +INSERT INTO nulls VALUES (0, NULL, -900, NULL, -9000000000, NULL, 1.65, NULL, '059', NULL, X'3B'); +INSERT INTO nulls VALUES (NULL, -10, NULL, -10000000, NULL, 495849600000, NULL, '60', NULL, X'3C', NULL); +INSERT INTO nulls VALUES (0, NULL, -1100, NULL, -11000000000, NULL, 1.596774193548387, NULL, '061', NULL, X'3D'); +INSERT INTO nulls VALUES (NULL, -12, NULL, -12000000, NULL, 496022400000, NULL, '62', NULL, X'3E3E3E', NULL); +INSERT INTO nulls VALUES (0, NULL, -1300, NULL, -13000000000, NULL, 1.546875, NULL, '063', NULL, X'3F'); +INSERT INTO nulls VALUES (NULL, -14, NULL, -14000000, NULL, 496195200000, NULL, '64', NULL, X'4040404040', NULL); +INSERT INTO nulls VALUES (0, NULL, -1500, NULL, -15000000000, NULL, 1.5, NULL, '065', NULL, X'41'); +INSERT INTO nulls VALUES (NULL, -16, NULL, -16000000, NULL, 496368000000, NULL, '66', NULL, X'4242', NULL); +INSERT INTO nulls VALUES (0, NULL, -1700, NULL, -17000000000, NULL, 1.4558823529411764, NULL, '067', NULL, X'43'); +INSERT INTO nulls VALUES (NULL, -18, NULL, -18000000, NULL, 496540800000, NULL, '68', NULL, X'44444444', NULL); +INSERT INTO nulls VALUES (0, NULL, -1900, NULL, -19000000000, NULL, 1.4142857142857144, NULL, '069', NULL, X'45'); +INSERT INTO nulls VALUES (NULL, -20, NULL, -20000000, NULL, 496713600000, NULL, '70', NULL, X'46', NULL); +INSERT INTO nulls VALUES (0, NULL, -2100, NULL, -21000000000, NULL, 1.375, NULL, '071', NULL, X'47'); +INSERT INTO nulls VALUES (NULL, -22, NULL, -22000000, NULL, 496886400000, NULL, '72', NULL, X'484848', NULL); +INSERT INTO nulls VALUES (0, NULL, -2300, NULL, -23000000000, NULL, 1.337837837837838, NULL, '073', NULL, X'49'); +INSERT INTO nulls VALUES (NULL, -24, NULL, -24000000, NULL, 497059200000, NULL, '74', NULL, X'4A4A4A4A4A', NULL); +INSERT INTO nulls VALUES (0, NULL, -2500, NULL, -25000000000, NULL, 1.3026315789473684, NULL, '075', NULL, X'4B'); +INSERT INTO nulls VALUES (NULL, -26, NULL, -26000000, NULL, 497232000000, NULL, '76', NULL, X'4C4C', NULL); +INSERT INTO nulls VALUES (0, NULL, -2700, NULL, -27000000000, NULL, 1.2692307692307692, NULL, '077', NULL, X'4D'); +INSERT INTO nulls VALUES (NULL, -28, NULL, -28000000, NULL, 497404800000, NULL, '78', NULL, X'4E4E4E4E', NULL); +INSERT INTO nulls VALUES (0, NULL, -2900, NULL, -29000000000, NULL, 1.2375, NULL, '079', NULL, X'4F'); +INSERT INTO nulls VALUES (NULL, -30, NULL, -30000000, NULL, 497577600000, NULL, '80', NULL, X'50', NULL); +INSERT INTO nulls VALUES (0, NULL, -3100, NULL, -31000000000, NULL, 1.2073170731707317, NULL, '081', NULL, X'51'); +INSERT INTO nulls VALUES (NULL, -32, NULL, -32000000, NULL, 497750400000, NULL, '82', NULL, X'525252', NULL); +INSERT INTO nulls VALUES (0, NULL, -3300, NULL, -33000000000, NULL, 1.1785714285714286, NULL, '083', NULL, X'53'); +INSERT INTO nulls VALUES (NULL, -34, NULL, -34000000, NULL, 497923200000, NULL, '84', NULL, X'5454545454', NULL); +INSERT INTO nulls VALUES (0, NULL, -3500, NULL, -35000000000, NULL, 1.1511627906976745, NULL, '085', NULL, X'55'); +INSERT INTO nulls VALUES (NULL, -36, NULL, -36000000, NULL, 498096000000, NULL, '86', NULL, X'5656', NULL); +INSERT INTO nulls VALUES (0, NULL, -3700, NULL, -37000000000, NULL, 1.125, NULL, '087', NULL, X'57'); +INSERT INTO nulls VALUES (NULL, -38, NULL, -38000000, NULL, 498268800000, NULL, '88', NULL, X'58585858', NULL); +INSERT INTO nulls VALUES (0, NULL, -3900, NULL, -39000000000, NULL, 1.1, NULL, '089', NULL, X'59'); +INSERT INTO nulls VALUES (NULL, -40, NULL, -40000000, NULL, 498441600000, NULL, '90', NULL, X'5A', NULL); +INSERT INTO nulls VALUES (0, NULL, -4100, NULL, -41000000000, NULL, 1.076086956521739, NULL, '091', NULL, X'5B'); +INSERT INTO nulls VALUES (NULL, -42, NULL, -42000000, NULL, 498614400000, NULL, '92', NULL, X'5C5C5C', NULL); +INSERT INTO nulls VALUES (0, NULL, -4300, NULL, -43000000000, NULL, 1.053191489361702, NULL, '093', NULL, X'5D'); +INSERT INTO nulls VALUES (NULL, -44, NULL, -44000000, NULL, 498787200000, NULL, '94', NULL, X'5E5E5E5E5E', NULL); +INSERT INTO nulls VALUES (0, NULL, -4500, NULL, -45000000000, NULL, 1.03125, NULL, '095', NULL, X'5F'); +INSERT INTO nulls VALUES (NULL, -46, NULL, -46000000, NULL, 498960000000, NULL, '96', NULL, X'6060', NULL); +INSERT INTO nulls VALUES (0, NULL, -4700, NULL, -47000000000, NULL, 1.010204081632653, NULL, '097', NULL, X'61'); +INSERT INTO nulls VALUES (NULL, -48, NULL, -48000000, NULL, 499132800000, NULL, '98', NULL, X'62626262', NULL); +COMMIT; diff --git a/parquet-generator/parquets.py b/parquet-generator/parquets.py index b6b75da..61596b7 100644 --- a/parquet-generator/parquets.py +++ b/parquet-generator/parquets.py @@ -50,6 +50,13 @@ def get_99_rows_types(): # pa.float32() ] +def name_of(type, i): + name = '{}_{}'.format(type, i) + name = name.replace('timestamp[ns]', 'ts') + name = name.replace('fixed_size_binary[1]', 'binary') + return name + + def write_parquet(file_name, rows, types, row_group_size): '''Create two parquets with columns we support.''' # pivot to be column major, create arrow structures @@ -59,13 +66,7 @@ def write_parquet(file_name, rows, types, row_group_size): col.append([row[i] for row in rows]) fields.append(pa.chunked_array(col, type=types[i])) - def name_of(i): - name = '{}_{}'.format(types[i], i) - name = name.replace('timestamp[ns]', 'ts') - name = name.replace('fixed_size_binary[1]', 'binary') - return name - - cols = [pa.Column.from_array(name_of(i), fields[i]) for i in range(len(fields))] + cols = [pa.Column.from_array(name_of(types[i], i), fields[i]) for i in range(len(fields))] table = pa.Table.from_arrays(cols) print('Writing {}'.format(file_name)) pq.write_table(table, @@ -128,6 +129,77 @@ def write_csv(file_name, rows): f.write(line + '\n') +def type_of(type): + if type == pa.bool_(): + return 'BOOLEAN' + elif type == pa.int8(): + return 'TINYINT' + elif type == pa.int16(): + return 'SMALLINT' + elif type == pa.int32(): + return 'INT' + elif type == pa.int64() or type == pa.timestamp('ns'): + return 'BIGINT' + elif type == pa.float64(): + return 'DOUBLE' + elif type == pa.string(): + return 'TEXT' + elif type == pa.binary(): + return 'BLOB' + elif type == pa.binary(1): + return 'BLOB' + else: + raise ValueError('unknown type: {}'.format(type)) + +def write_sql(file_name, rows, types): + table_name = file_name.replace('.sql', '').replace('-', '_') + print('Writing {} [{}]'.format(file_name, table_name)) + + with open(file_name, 'w') as f: + f.write('BEGIN;') + f.write('DROP TABLE IF EXISTS {};\n'.format(table_name)) + f.write('CREATE TABLE {} ('.format(table_name)) + for i, col in enumerate(types): + if i > 0: + f.write(', '); + + col_name = name_of(col, i) + f.write('{} {}'.format(name_of(col, i), type_of(col))) + f.write(');\n') + + for row in rows: + f.write('INSERT INTO {} VALUES ('.format(table_name)) + line = '' + for i, col in enumerate(row): + if i > 0: + line += ', ' + + if col == True: + line += '1' + elif col == False: + line += '0' + elif col is None: + line += 'NULL' + elif isinstance(col, bytes): + entry = r"X'" + for b in col: + entry += '%0.2X' % b + + entry += "'" + line += entry + elif isinstance(col, datetime): + line += str(1000 * int(col.timestamp())) + elif isinstance(col, str): + line += "'{}'".format(col) + else: + line += str(col) + + f.write(line) + f.write(');\n') + f.write('COMMIT;\n') + + + def main(): '''Entrypoint.''' rows = make_99_rows() @@ -137,6 +209,7 @@ def main(): write_csv('no-nulls.csv', rows) write_parquet('99-rows-10.parquet', rows, types, row_group_size=10) write_parquet('99-rows-99.parquet', rows, types, row_group_size=1) + write_sql('no-nulls.sql', rows, types) for i in range(len(rows)): for j in range(len(rows[i])): @@ -146,6 +219,7 @@ def main(): write_parquet('99-rows-nulls-10.parquet', rows, types,row_group_size=10) write_parquet('99-rows-nulls-1.parquet', rows, types,row_group_size=1) write_csv('nulls.csv', rows) + write_sql('nulls.sql', rows, types) write_unsupported_parquets() diff --git a/tests/create-queries-from-templates b/tests/create-queries-from-templates index f4f64d7..124ea45 100755 --- a/tests/create-queries-from-templates +++ b/tests/create-queries-from-templates @@ -6,8 +6,8 @@ from glob import glob import re import itertools -NULL_TOKENS = ['nulls1', 'nulls2', 'nulls3'] -NO_NULL_TOKENS = ['no_nulls1', 'no_nulls2', 'no_nulls3'] +NULL_TOKENS = ['nulls', 'nulls1', 'nulls2', 'nulls3'] +NO_NULL_TOKENS = ['no_nulls', 'no_nulls1', 'no_nulls2', 'no_nulls3'] TOKEN_SET = NULL_TOKENS + NO_NULL_TOKENS diff --git a/tests/test-queries b/tests/test-queries index d73786f..f8b0f51 100755 --- a/tests/test-queries +++ b/tests/test-queries @@ -12,12 +12,12 @@ run_query() { .load parquet/libparquet .testcase $basename .bail on -CREATE VIRTUAL TABLE nulls1 USING parquet('$root/parquet-generator/99-rows-nulls-1.parquet'); -CREATE VIRTUAL TABLE nulls2 USING parquet('$root/parquet-generator/99-rows-nulls-10.parquet'); -CREATE VIRTUAL TABLE nulls3 USING parquet('$root/parquet-generator/99-rows-nulls-99.parquet'); -CREATE VIRTUAL TABLE no_nulls1 USING parquet('$root/parquet-generator/99-rows-1.parquet'); -CREATE VIRTUAL TABLE no_nulls2 USING parquet('$root/parquet-generator/99-rows-10.parquet'); -CREATE VIRTUAL TABLE no_nulls3 USING parquet('$root/parquet-generator/99-rows-99.parquet'); +CREATE VIRTUAL TABLE IF NOT EXISTS nulls1 USING parquet('$root/parquet-generator/99-rows-nulls-1.parquet'); +CREATE VIRTUAL TABLE IF NOT EXISTS nulls2 USING parquet('$root/parquet-generator/99-rows-nulls-10.parquet'); +CREATE VIRTUAL TABLE IF NOT EXISTS nulls3 USING parquet('$root/parquet-generator/99-rows-nulls-99.parquet'); +CREATE VIRTUAL TABLE IF NOT EXISTS no_nulls1 USING parquet('$root/parquet-generator/99-rows-1.parquet'); +CREATE VIRTUAL TABLE IF NOT EXISTS no_nulls2 USING parquet('$root/parquet-generator/99-rows-10.parquet'); +CREATE VIRTUAL TABLE IF NOT EXISTS no_nulls3 USING parquet('$root/parquet-generator/99-rows-99.parquet'); $query; .output EOF @@ -42,9 +42,12 @@ main() { exit 1 fi + cat "$root"/parquet-generator/*.sql > "$root"/testcase-bootstrap.sql + rm test.db + "$root"/sqlite/sqlite3 test.db -init "$root"/testcase-bootstrap.sql < /dev/null if [ ! -v NO_DEBUG ] && [ "$(cat testcases.txt | wc -l)" == "1" ]; then set -x - gdb -ex run --args "$root"/sqlite/sqlite3 -init testcase-cmds.txt + gdb -ex run --args "$root"/sqlite/sqlite3 test.db -init testcase-cmds.txt else while read -r file; do echo "Testing: $file" @@ -52,7 +55,7 @@ main() { tail -n+2 "$file" > testcase-expected.txt run_query "$file" "$query" > testcase-cmds.txt - if ! "$root"/sqlite/sqlite3 -init testcase-cmds.txt < /dev/null > testcase-stdout.txt 2> testcase-stderr.txt; then + if ! "$root"/sqlite/sqlite3 test.db -init testcase-cmds.txt < /dev/null > testcase-stdout.txt 2> testcase-stderr.txt; then echo "...FAILED; check testcase-{out,err}.txt" >&2 exit 1 fi