diff --git a/parquet-generator/100-rows-1.parquet b/parquet-generator/100-rows-1.parquet index fe4fa70..152ae7f 100644 Binary files a/parquet-generator/100-rows-1.parquet and b/parquet-generator/100-rows-1.parquet differ diff --git a/parquet-generator/100-rows-10.parquet b/parquet-generator/100-rows-10.parquet index aff0f9b..e8e162b 100644 Binary files a/parquet-generator/100-rows-10.parquet and b/parquet-generator/100-rows-10.parquet differ diff --git a/parquet-generator/100-rows-nulls.parquet b/parquet-generator/100-rows-nulls.parquet index 08c5b78..3c6b6e6 100644 Binary files a/parquet-generator/100-rows-nulls.parquet and b/parquet-generator/100-rows-nulls.parquet differ diff --git a/parquet-generator/create-pg-db b/parquet-generator/create-pg-db new file mode 100755 index 0000000..9a08e0a --- /dev/null +++ b/parquet-generator/create-pg-db @@ -0,0 +1,55 @@ +#!/bin/bash + +set -euo pipefail + +export PGUSER=postgres + +dropdb parquet || true +createdb parquet + +here=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")") + +psql parquet << EOF +CREATE TABLE no_nulls( + rowid INT2, + bool_0 INT2, + int8_1 INT2, + int16_2 INT2, + int32_3 INT4, + int64_4 INT8, + ts_5 int8, + double_6 DOUBLE PRECISION, + string_7 TEXT, + string_8 TEXT, + binary_9 TEXT, + binary_10 TEXT +); + +CREATE TABLE nulls (LIKE no_nulls); + +COPY no_nulls FROM '$here/no-nulls.csv'; +COPY nulls FROM '$here/nulls.csv'; + +CREATE OR REPLACE FUNCTION quote(TEXT) RETURNS TEXT AS \$\$ + BEGIN + RETURN \$1; + END; +\$\$ LANGUAGE 'plpgsql' IMMUTABLE; + +CREATE OR REPLACE FUNCTION datetime(INT8, TEXT) RETURNS TEXT AS \$\$ + BEGIN + RETURN to_timestamp(\$1) AT TIME ZONE 'UTC'; + END; +\$\$ LANGUAGE 'plpgsql' IMMUTABLE; + +CREATE OR REPLACE FUNCTION printf(TEXT, DOUBLE PRECISION) RETURNS TEXT AS \$\$ + BEGIN + RETURN CAST(\$2 AS DECIMAL(10, 4))::TEXT; + END; +\$\$ LANGUAGE 'plpgsql' IMMUTABLE; + + + +--COPY (SELECT rowid, bool_0, int8_1, int16_2, int32_3, int64_4, datetime(ts_5 / 1000, 'unixepoch'), double_6, string_7, string_8, quote(binary_9), quote(binary_10) FROM nulls LIMIT 3) TO STDOUT WITH (DELIMITER '|', NULL ''); +COPY (SELECT rowid, bool_0, int8_1, int16_2, int32_3, int64_4, datetime(ts_5 / 1000, 'unixepoch'), printf('%.4f', double_6) AS double_6, string_7, string_8, quote(binary_9), quote(binary_10) FROM no_nulls) TO STDOUT WITH (DELIMITER '|', NULL ''); +EOF diff --git a/parquet-generator/no-nulls.csv b/parquet-generator/no-nulls.csv new file mode 100644 index 0000000..a68c680 --- /dev/null +++ b/parquet-generator/no-nulls.csv @@ -0,0 +1,100 @@ +0 1 50 5000 50000000 50000000000 490665600000 100.0 0 000 X'00' X'00' +1 0 49 4900 49000000 49000000000 490752000000 50.0 1 001 X'0101' X'01' +2 1 48 4800 48000000 48000000000 490838400000 33.333333333333336 2 002 X'020202' X'02' +3 0 47 4700 47000000 47000000000 490924800000 25.0 3 003 X'03030303' X'03' +4 1 46 4600 46000000 46000000000 491011200000 20.0 4 004 X'0404040404' X'04' +5 0 45 4500 45000000 45000000000 491097600000 16.666666666666668 5 005 X'05' X'05' +6 1 44 4400 44000000 44000000000 491184000000 14.285714285714286 6 006 X'0606' X'06' +7 0 43 4300 43000000 43000000000 491270400000 12.5 7 007 X'070707' X'07' +8 1 42 4200 42000000 42000000000 491356800000 11.11111111111111 8 008 X'08080808' X'08' +9 0 41 4100 41000000 41000000000 491443200000 10.0 9 009 X'0909090909' X'09' +10 1 40 4000 40000000 40000000000 491529600000 9.090909090909092 10 010 X'0A' X'0A' +11 0 39 3900 39000000 39000000000 491616000000 8.333333333333334 11 011 X'0B0B' X'0B' +12 1 38 3800 38000000 38000000000 491702400000 7.6923076923076925 12 012 X'0C0C0C' X'0C' +13 0 37 3700 37000000 37000000000 491788800000 7.142857142857143 13 013 X'0D0D0D0D' X'0D' +14 1 36 3600 36000000 36000000000 491875200000 6.666666666666667 14 014 X'0E0E0E0E0E' X'0E' +15 0 35 3500 35000000 35000000000 491961600000 6.25 15 015 X'0F' X'0F' +16 1 34 3400 34000000 34000000000 492048000000 5.882352941176471 16 016 X'1010' X'10' +17 0 33 3300 33000000 33000000000 492134400000 5.555555555555555 17 017 X'111111' X'11' +18 1 32 3200 32000000 32000000000 492220800000 5.2631578947368425 18 018 X'12121212' X'12' +19 0 31 3100 31000000 31000000000 492307200000 5.0 19 019 X'1313131313' X'13' +20 1 30 3000 30000000 30000000000 492393600000 4.761904761904762 20 020 X'14' X'14' +21 0 29 2900 29000000 29000000000 492480000000 4.545454545454546 21 021 X'1515' X'15' +22 1 28 2800 28000000 28000000000 492566400000 4.3478260869565215 22 022 X'161616' X'16' +23 0 27 2700 27000000 27000000000 492652800000 4.166666666666667 23 023 X'17171717' X'17' +24 1 26 2600 26000000 26000000000 492739200000 4.0 24 024 X'1818181818' X'18' +25 0 25 2500 25000000 25000000000 492825600000 3.8461538461538463 25 025 X'19' X'19' +26 1 24 2400 24000000 24000000000 492912000000 3.7037037037037037 26 026 X'1A1A' X'1A' +27 0 23 2300 23000000 23000000000 492998400000 3.5714285714285716 27 027 X'1B1B1B' X'1B' +28 1 22 2200 22000000 22000000000 493084800000 3.4482758620689653 28 028 X'1C1C1C1C' X'1C' +29 0 21 2100 21000000 21000000000 493171200000 3.3333333333333335 29 029 X'1D1D1D1D1D' X'1D' +30 1 20 2000 20000000 20000000000 493257600000 3.225806451612903 30 030 X'1E' X'1E' +31 0 19 1900 19000000 19000000000 493344000000 3.125 31 031 X'1F1F' X'1F' +32 1 18 1800 18000000 18000000000 493430400000 3.0303030303030303 32 032 X'202020' X'20' +33 0 17 1700 17000000 17000000000 493516800000 2.9411764705882355 33 033 X'21212121' X'21' +34 1 16 1600 16000000 16000000000 493603200000 2.857142857142857 34 034 X'2222222222' X'22' +35 0 15 1500 15000000 15000000000 493689600000 2.7777777777777777 35 035 X'23' X'23' +36 1 14 1400 14000000 14000000000 493776000000 2.7027027027027026 36 036 X'2424' X'24' +37 0 13 1300 13000000 13000000000 493862400000 2.6315789473684212 37 037 X'252525' X'25' +38 1 12 1200 12000000 12000000000 493948800000 2.5641025641025643 38 038 X'26262626' X'26' +39 0 11 1100 11000000 11000000000 494035200000 2.5 39 039 X'2727272727' X'27' +40 1 10 1000 10000000 10000000000 494121600000 2.4390243902439024 40 040 X'28' X'28' +41 0 9 900 9000000 9000000000 494208000000 2.380952380952381 41 041 X'2929' X'29' +42 1 8 800 8000000 8000000000 494294400000 2.3255813953488373 42 042 X'2A2A2A' X'2A' +43 0 7 700 7000000 7000000000 494380800000 2.272727272727273 43 043 X'2B2B2B2B' X'2B' +44 1 6 600 6000000 6000000000 494467200000 2.2222222222222223 44 044 X'2C2C2C2C2C' X'2C' +45 0 5 500 5000000 5000000000 494553600000 2.1739130434782608 45 045 X'2D' X'2D' +46 1 4 400 4000000 4000000000 494640000000 2.127659574468085 46 046 X'2E2E' X'2E' +47 0 3 300 3000000 3000000000 494726400000 2.0833333333333335 47 047 X'2F2F2F' X'2F' +48 1 2 200 2000000 2000000000 494812800000 2.0408163265306123 48 048 X'30303030' X'30' +49 0 1 100 1000000 1000000000 494899200000 2.0 49 049 X'3131313131' X'31' +50 1 0 0 0 0 494985600000 1.9607843137254901 50 050 X'32' X'32' +51 0 -1 -100 -1000000 -1000000000 495072000000 1.9230769230769231 51 051 X'3333' X'33' +52 1 -2 -200 -2000000 -2000000000 495158400000 1.8867924528301887 52 052 X'343434' X'34' +53 0 -3 -300 -3000000 -3000000000 495244800000 1.8518518518518519 53 053 X'35353535' X'35' +54 1 -4 -400 -4000000 -4000000000 495331200000 1.8181818181818181 54 054 X'3636363636' X'36' +55 0 -5 -500 -5000000 -5000000000 495417600000 1.7857142857142858 55 055 X'37' X'37' +56 1 -6 -600 -6000000 -6000000000 495504000000 1.7543859649122806 56 056 X'3838' X'38' +57 0 -7 -700 -7000000 -7000000000 495590400000 1.7241379310344827 57 057 X'393939' X'39' +58 1 -8 -800 -8000000 -8000000000 495676800000 1.694915254237288 58 058 X'3A3A3A3A' X'3A' +59 0 -9 -900 -9000000 -9000000000 495763200000 1.6666666666666667 59 059 X'3B3B3B3B3B' X'3B' +60 1 -10 -1000 -10000000 -10000000000 495849600000 1.639344262295082 60 060 X'3C' X'3C' +61 0 -11 -1100 -11000000 -11000000000 495936000000 1.6129032258064515 61 061 X'3D3D' X'3D' +62 1 -12 -1200 -12000000 -12000000000 496022400000 1.5873015873015872 62 062 X'3E3E3E' X'3E' +63 0 -13 -1300 -13000000 -13000000000 496108800000 1.5625 63 063 X'3F3F3F3F' X'3F' +64 1 -14 -1400 -14000000 -14000000000 496195200000 1.5384615384615385 64 064 X'4040404040' X'40' +65 0 -15 -1500 -15000000 -15000000000 496281600000 1.5151515151515151 65 065 X'41' X'41' +66 1 -16 -1600 -16000000 -16000000000 496368000000 1.492537313432836 66 066 X'4242' X'42' +67 0 -17 -1700 -17000000 -17000000000 496454400000 1.4705882352941178 67 067 X'434343' X'43' +68 1 -18 -1800 -18000000 -18000000000 496540800000 1.4492753623188406 68 068 X'44444444' X'44' +69 0 -19 -1900 -19000000 -19000000000 496627200000 1.4285714285714286 69 069 X'4545454545' X'45' +70 1 -20 -2000 -20000000 -20000000000 496713600000 1.408450704225352 70 070 X'46' X'46' +71 0 -21 -2100 -21000000 -21000000000 496800000000 1.3888888888888888 71 071 X'4747' X'47' +72 1 -22 -2200 -22000000 -22000000000 496886400000 1.36986301369863 72 072 X'484848' X'48' +73 0 -23 -2300 -23000000 -23000000000 496972800000 1.3513513513513513 73 073 X'49494949' X'49' +74 1 -24 -2400 -24000000 -24000000000 497059200000 1.3333333333333333 74 074 X'4A4A4A4A4A' X'4A' +75 0 -25 -2500 -25000000 -25000000000 497145600000 1.3157894736842106 75 075 X'4B' X'4B' +76 1 -26 -2600 -26000000 -26000000000 497232000000 1.2987012987012987 76 076 X'4C4C' X'4C' +77 0 -27 -2700 -27000000 -27000000000 497318400000 1.2820512820512822 77 077 X'4D4D4D' X'4D' +78 1 -28 -2800 -28000000 -28000000000 497404800000 1.2658227848101267 78 078 X'4E4E4E4E' X'4E' +79 0 -29 -2900 -29000000 -29000000000 497491200000 1.25 79 079 X'4F4F4F4F4F' X'4F' +80 1 -30 -3000 -30000000 -30000000000 497577600000 1.2345679012345678 80 080 X'50' X'50' +81 0 -31 -3100 -31000000 -31000000000 497664000000 1.2195121951219512 81 081 X'5151' X'51' +82 1 -32 -3200 -32000000 -32000000000 497750400000 1.2048192771084338 82 082 X'525252' X'52' +83 0 -33 -3300 -33000000 -33000000000 497836800000 1.1904761904761905 83 083 X'53535353' X'53' +84 1 -34 -3400 -34000000 -34000000000 497923200000 1.1764705882352942 84 084 X'5454545454' X'54' +85 0 -35 -3500 -35000000 -35000000000 498009600000 1.1627906976744187 85 085 X'55' X'55' +86 1 -36 -3600 -36000000 -36000000000 498096000000 1.1494252873563218 86 086 X'5656' X'56' +87 0 -37 -3700 -37000000 -37000000000 498182400000 1.1363636363636365 87 087 X'575757' X'57' +88 1 -38 -3800 -38000000 -38000000000 498268800000 1.1235955056179776 88 088 X'58585858' X'58' +89 0 -39 -3900 -39000000 -39000000000 498355200000 1.1111111111111112 89 089 X'5959595959' X'59' +90 1 -40 -4000 -40000000 -40000000000 498441600000 1.098901098901099 90 090 X'5A' X'5A' +91 0 -41 -4100 -41000000 -41000000000 498528000000 1.0869565217391304 91 091 X'5B5B' X'5B' +92 1 -42 -4200 -42000000 -42000000000 498614400000 1.075268817204301 92 092 X'5C5C5C' X'5C' +93 0 -43 -4300 -43000000 -43000000000 498700800000 1.0638297872340425 93 093 X'5D5D5D5D' X'5D' +94 1 -44 -4400 -44000000 -44000000000 498787200000 1.0526315789473684 94 094 X'5E5E5E5E5E' X'5E' +95 0 -45 -4500 -45000000 -45000000000 498873600000 1.0416666666666667 95 095 X'5F' X'5F' +96 1 -46 -4600 -46000000 -46000000000 498960000000 1.0309278350515463 96 096 X'6060' X'60' +97 0 -47 -4700 -47000000 -47000000000 499046400000 1.0204081632653061 97 097 X'616161' X'61' +98 1 -48 -4800 -48000000 -48000000000 499132800000 1.0101010101010102 98 098 X'62626262' X'62' +99 0 -49 -4900 -49000000 -49000000000 499219200000 1 99 099 X'6363636363' X'63' diff --git a/parquet-generator/nulls.csv b/parquet-generator/nulls.csv new file mode 100644 index 0000000..233a357 --- /dev/null +++ b/parquet-generator/nulls.csv @@ -0,0 +1,100 @@ +0 1 50 5000 50000000 50000000000 490665600000 100.0 0 000 X'00' X'00' +1 0 49 4900 49000000 49000000000 490752000000 50.0 1 001 X'0101' X'01' +2 1 48 4800 48000000 48000000000 490838400000 33.333333333333336 2 002 X'020202' X'02' +3 0 47 4700 47000000 47000000000 490924800000 25.0 3 003 X'03030303' X'03' +4 1 46 4600 46000000 46000000000 491011200000 20.0 4 004 X'0404040404' X'04' +5 0 45 4500 45000000 45000000000 491097600000 16.666666666666668 5 005 X'05' X'05' +6 1 44 4400 44000000 44000000000 491184000000 14.285714285714286 6 006 X'0606' X'06' +7 0 43 4300 43000000 43000000000 491270400000 12.5 7 007 X'070707' X'07' +8 1 42 4200 42000000 42000000000 491356800000 11.11111111111111 8 008 X'08080808' X'08' +9 0 41 4100 41000000 41000000000 491443200000 10.0 9 009 X'0909090909' X'09' +10 \N \N \N \N \N \N \N \N \N \N \N +11 \N \N \N \N \N \N \N \N \N \N \N +12 \N \N \N \N \N \N \N \N \N \N \N +13 \N \N \N \N \N \N \N \N \N \N \N +14 \N \N \N \N \N \N \N \N \N \N \N +15 \N \N \N \N \N \N \N \N \N \N \N +16 \N \N \N \N \N \N \N \N \N \N \N +17 \N \N \N \N \N \N \N \N \N \N \N +18 \N \N \N \N \N \N \N \N \N \N \N +19 \N \N \N \N \N \N \N \N \N \N \N +20 \N 30 \N 30000000 \N 492393600000 \N 20 \N X'14' \N +21 0 \N 2900 \N 29000000000 \N 4.545454545454546 \N 021 \N X'15' +22 \N 28 \N 28000000 \N 492566400000 \N 22 \N X'161616' \N +23 0 \N 2700 \N 27000000000 \N 4.166666666666667 \N 023 \N X'17' +24 \N 26 \N 26000000 \N 492739200000 \N 24 \N X'1818181818' \N +25 0 \N 2500 \N 25000000000 \N 3.8461538461538463 \N 025 \N X'19' +26 \N 24 \N 24000000 \N 492912000000 \N 26 \N X'1A1A' \N +27 0 \N 2300 \N 23000000000 \N 3.5714285714285716 \N 027 \N X'1B' +28 \N 22 \N 22000000 \N 493084800000 \N 28 \N X'1C1C1C1C' \N +29 0 \N 2100 \N 21000000000 \N 3.3333333333333335 \N 029 \N X'1D' +30 \N 20 \N 20000000 \N 493257600000 \N 30 \N X'1E' \N +31 0 \N 1900 \N 19000000000 \N 3.125 \N 031 \N X'1F' +32 \N 18 \N 18000000 \N 493430400000 \N 32 \N X'202020' \N +33 0 \N 1700 \N 17000000000 \N 2.9411764705882355 \N 033 \N X'21' +34 \N 16 \N 16000000 \N 493603200000 \N 34 \N X'2222222222' \N +35 0 \N 1500 \N 15000000000 \N 2.7777777777777777 \N 035 \N X'23' +36 \N 14 \N 14000000 \N 493776000000 \N 36 \N X'2424' \N +37 0 \N 1300 \N 13000000000 \N 2.6315789473684212 \N 037 \N X'25' +38 \N 12 \N 12000000 \N 493948800000 \N 38 \N X'26262626' \N +39 0 \N 1100 \N 11000000000 \N 2.5 \N 039 \N X'27' +40 \N 10 \N 10000000 \N 494121600000 \N 40 \N X'28' \N +41 0 \N 900 \N 9000000000 \N 2.380952380952381 \N 041 \N X'29' +42 \N 8 \N 8000000 \N 494294400000 \N 42 \N X'2A2A2A' \N +43 0 \N 700 \N 7000000000 \N 2.272727272727273 \N 043 \N X'2B' +44 \N 6 \N 6000000 \N 494467200000 \N 44 \N X'2C2C2C2C2C' \N +45 0 \N 500 \N 5000000000 \N 2.1739130434782608 \N 045 \N X'2D' +46 \N 4 \N 4000000 \N 494640000000 \N 46 \N X'2E2E' \N +47 0 \N 300 \N 3000000000 \N 2.0833333333333335 \N 047 \N X'2F' +48 \N 2 \N 2000000 \N 494812800000 \N 48 \N X'30303030' \N +49 0 \N 100 \N 1000000000 \N 2.0 \N 049 \N X'31' +50 \N 0 \N 0 \N 494985600000 \N 50 \N X'32' \N +51 0 \N -100 \N -1000000000 \N 1.9230769230769231 \N 051 \N X'33' +52 \N -2 \N -2000000 \N 495158400000 \N 52 \N X'343434' \N +53 0 \N -300 \N -3000000000 \N 1.8518518518518519 \N 053 \N X'35' +54 \N -4 \N -4000000 \N 495331200000 \N 54 \N X'3636363636' \N +55 0 \N -500 \N -5000000000 \N 1.7857142857142858 \N 055 \N X'37' +56 \N -6 \N -6000000 \N 495504000000 \N 56 \N X'3838' \N +57 0 \N -700 \N -7000000000 \N 1.7241379310344827 \N 057 \N X'39' +58 \N -8 \N -8000000 \N 495676800000 \N 58 \N X'3A3A3A3A' \N +59 0 \N -900 \N -9000000000 \N 1.6666666666666667 \N 059 \N X'3B' +60 \N -10 \N -10000000 \N 495849600000 \N 60 \N X'3C' \N +61 0 \N -1100 \N -11000000000 \N 1.6129032258064515 \N 061 \N X'3D' +62 \N -12 \N -12000000 \N 496022400000 \N 62 \N X'3E3E3E' \N +63 0 \N -1300 \N -13000000000 \N 1.5625 \N 063 \N X'3F' +64 \N -14 \N -14000000 \N 496195200000 \N 64 \N X'4040404040' \N +65 0 \N -1500 \N -15000000000 \N 1.5151515151515151 \N 065 \N X'41' +66 \N -16 \N -16000000 \N 496368000000 \N 66 \N X'4242' \N +67 0 \N -1700 \N -17000000000 \N 1.4705882352941178 \N 067 \N X'43' +68 \N -18 \N -18000000 \N 496540800000 \N 68 \N X'44444444' \N +69 0 \N -1900 \N -19000000000 \N 1.4285714285714286 \N 069 \N X'45' +70 \N -20 \N -20000000 \N 496713600000 \N 70 \N X'46' \N +71 0 \N -2100 \N -21000000000 \N 1.3888888888888888 \N 071 \N X'47' +72 \N -22 \N -22000000 \N 496886400000 \N 72 \N X'484848' \N +73 0 \N -2300 \N -23000000000 \N 1.3513513513513513 \N 073 \N X'49' +74 \N -24 \N -24000000 \N 497059200000 \N 74 \N X'4A4A4A4A4A' \N +75 0 \N -2500 \N -25000000000 \N 1.3157894736842106 \N 075 \N X'4B' +76 \N -26 \N -26000000 \N 497232000000 \N 76 \N X'4C4C' \N +77 0 \N -2700 \N -27000000000 \N 1.2820512820512822 \N 077 \N X'4D' +78 \N -28 \N -28000000 \N 497404800000 \N 78 \N X'4E4E4E4E' \N +79 0 \N -2900 \N -29000000000 \N 1.25 \N 079 \N X'4F' +80 \N -30 \N -30000000 \N 497577600000 \N 80 \N X'50' \N +81 0 \N -3100 \N -31000000000 \N 1.2195121951219512 \N 081 \N X'51' +82 \N -32 \N -32000000 \N 497750400000 \N 82 \N X'525252' \N +83 0 \N -3300 \N -33000000000 \N 1.1904761904761905 \N 083 \N X'53' +84 \N -34 \N -34000000 \N 497923200000 \N 84 \N X'5454545454' \N +85 0 \N -3500 \N -35000000000 \N 1.1627906976744187 \N 085 \N X'55' +86 \N -36 \N -36000000 \N 498096000000 \N 86 \N X'5656' \N +87 0 \N -3700 \N -37000000000 \N 1.1363636363636365 \N 087 \N X'57' +88 \N -38 \N -38000000 \N 498268800000 \N 88 \N X'58585858' \N +89 0 \N -3900 \N -39000000000 \N 1.1111111111111112 \N 089 \N X'59' +90 \N -40 \N -40000000 \N 498441600000 \N 90 \N X'5A' \N +91 0 \N -4100 \N -41000000000 \N 1.0869565217391304 \N 091 \N X'5B' +92 \N -42 \N -42000000 \N 498614400000 \N 92 \N X'5C5C5C' \N +93 0 \N -4300 \N -43000000000 \N 1.0638297872340425 \N 093 \N X'5D' +94 \N -44 \N -44000000 \N 498787200000 \N 94 \N X'5E5E5E5E5E' \N +95 0 \N -4500 \N -45000000000 \N 1.0416666666666667 \N 095 \N X'5F' +96 \N -46 \N -46000000 \N 498960000000 \N 96 \N X'6060' \N +97 0 \N -4700 \N -47000000000 \N 1.0204081632653061 \N 097 \N X'61' +98 \N -48 \N -48000000 \N 499132800000 \N 98 \N X'62626262' \N +99 0 \N -4900 \N -49000000000 \N 1 \N 099 \N X'63' diff --git a/parquet-generator/parquets.py b/parquet-generator/parquets.py index df41da4..88ee10d 100644 --- a/parquet-generator/parquets.py +++ b/parquet-generator/parquets.py @@ -1,4 +1,4 @@ -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone import re import pyarrow as pa @@ -12,8 +12,8 @@ def make_100_rows(): ba_fixed = bytearray() ba_fixed.append(i) ba_variable = bytearray() - for j in range(i): - ba_variable.append(j) + for j in range(1 + i % 5): + ba_variable.append(i) row = [] # BOOLEAN, INT32, INT64, INT96, DOUBLE, BYTE_ARRAY, FLOAT row.append(i % 2 == 0) # BOOLEAN @@ -21,7 +21,7 @@ def make_100_rows(): row.append(100 * (50 - i)) # INT32/INT16 row.append(1000 * 1000 * (50 - i)) # INT32/INT32 row.append(1000 * 1000 * 1000 * (50 - i)) # INT64/INT64 - row.append(datetime(1985, 7, 20) + timedelta(days=i)) # INT96 + row.append(datetime(1985, 7, 20, tzinfo=timezone.utc) + timedelta(days=i)) # INT96 row.append(100.0 / (i + 1)) # DOUBLE row.append(str(i)) # BYTE_ARRAY/UTF8 row.append('{:03}'.format(i)), # BYTE_ARRAY/UTF8 @@ -95,12 +95,46 @@ def write_unsupported_parquets(): write_parquet(file_name, [], [type], row_group_size=1) +def write_csv(file_name, rows): + r'''Write a TSV that can be imported to Postgres. + + Use "\N" for NULLs, tab literal for field separator.''' + print('Writing {}'.format(file_name)) + with open(file_name, 'w') as f: + for rowid, row in enumerate(rows): + line = str(rowid) + for col in row: + line += '\t' + + if col == True: + line += '1' + elif col == False: + line += '0' + elif col is None: + line += r'\N' + elif isinstance(col, bytes): + # Here we cheat and serialize a string that matches the output of + # quote(binary_field) in SQLite + entry = r"X'" + for b in col: + entry += '%0.2X' % b + + entry += "'" + line += entry + elif isinstance(col, datetime): + line += str(1000 * int(col.timestamp())) + else: + line += str(col) + + f.write(line + '\n') + def main(): '''Entrypoint.''' rows = make_100_rows() types = get_100_rows_types() write_parquet('100-rows-1.parquet', rows, types, row_group_size=100) + write_csv('no-nulls.csv', rows) write_parquet('100-rows-10.parquet', rows, types, row_group_size=10) for i in range(len(rows)): @@ -108,6 +142,7 @@ def main(): if (i >= 10 and i <= 19) or (i >= 20 and (i + j) % 2 == 0): rows[i][j] = None write_parquet('100-rows-nulls.parquet', rows, types,row_group_size=10) + write_csv('nulls.csv', rows) write_unsupported_parquets() diff --git a/tests/queries/001-table-scan-1-rowgroup.sql b/tests/queries/001-table-scan-1-rowgroup.sql index a150a5b..e4ecc08 100644 --- a/tests/queries/001-table-scan-1-rowgroup.sql +++ b/tests/queries/001-table-scan-1-rowgroup.sql @@ -1,103 +1,102 @@ 100-rows-1.parquet -select * from test; -1|50|5000|50000000|50000000000|490665600000|100.0|0|000|| -0|49|4900|49000000|49000000000|490752000000|50.0|1|001|| -1|48|4800|48000000|48000000000|490838400000|33.3333333333333|2|002|| -0|47|4700|47000000|47000000000|490924800000|25.0|3|003|| -1|46|4600|46000000|46000000000|491011200000|20.0|4|004|| -0|45|4500|45000000|45000000000|491097600000|16.6666666666667|5|005|| -1|44|4400|44000000|44000000000|491184000000|14.2857142857143|6|006|| -0|43|4300|43000000|43000000000|491270400000|12.5|7|007|| -1|42|4200|42000000|42000000000|491356800000|11.1111111111111|8|008|| -0|41|4100|41000000|41000000000|491443200000|10.0|9|009|| -1|40|4000|40000000|40000000000|491529600000|9.09090909090909|10|010|| - -0|39|3900|39000000|39000000000|491616000000|8.33333333333333|11|011|| -1|38|3800|38000000|38000000000|491702400000|7.69230769230769|12|012|| -0|37|3700|37000000|37000000000|491788800000|7.14285714285714|13|013|| -1|36|3600|36000000|36000000000|491875200000|6.66666666666667|14|014|| -0|35|3500|35000000|35000000000|491961600000|6.25|15|015|| -1|34|3400|34000000|34000000000|492048000000|5.88235294117647|16|016|| -0|33|3300|33000000|33000000000|492134400000|5.55555555555556|17|017|| -1|32|3200|32000000|32000000000|492220800000|5.26315789473684|18|018|| -0|31|3100|31000000|31000000000|492307200000|5.0|19|019|| -1|30|3000|30000000|30000000000|492393600000|4.76190476190476|20|020|| -0|29|2900|29000000|29000000000|492480000000|4.54545454545455|21|021|| -1|28|2800|28000000|28000000000|492566400000|4.34782608695652|22|022|| -0|27|2700|27000000|27000000000|492652800000|4.16666666666667|23|023|| -1|26|2600|26000000|26000000000|492739200000|4.0|24|024|| -0|25|2500|25000000|25000000000|492825600000|3.84615384615385|25|025|| -1|24|2400|24000000|24000000000|492912000000|3.7037037037037|26|026|| -0|23|2300|23000000|23000000000|492998400000|3.57142857142857|27|027|| -1|22|2200|22000000|22000000000|493084800000|3.44827586206897|28|028|| -0|21|2100|21000000|21000000000|493171200000|3.33333333333333|29|029|| -1|20|2000|20000000|20000000000|493257600000|3.2258064516129|30|030|| -0|19|1900|19000000|19000000000|493344000000|3.125|31|031|| -1|18|1800|18000000|18000000000|493430400000|3.03030303030303|32|032|| -0|17|1700|17000000|17000000000|493516800000|2.94117647058824|33|033||! -1|16|1600|16000000|16000000000|493603200000|2.85714285714286|34|034||" -0|15|1500|15000000|15000000000|493689600000|2.77777777777778|35|035||# -1|14|1400|14000000|14000000000|493776000000|2.7027027027027|36|036||$ -0|13|1300|13000000|13000000000|493862400000|2.63157894736842|37|037||% -1|12|1200|12000000|12000000000|493948800000|2.56410256410256|38|038||& -0|11|1100|11000000|11000000000|494035200000|2.5|39|039||' -1|10|1000|10000000|10000000000|494121600000|2.4390243902439|40|040||( -0|9|900|9000000|9000000000|494208000000|2.38095238095238|41|041||) -1|8|800|8000000|8000000000|494294400000|2.32558139534884|42|042||* -0|7|700|7000000|7000000000|494380800000|2.27272727272727|43|043||+ -1|6|600|6000000|6000000000|494467200000|2.22222222222222|44|044||, -0|5|500|5000000|5000000000|494553600000|2.17391304347826|45|045||- -1|4|400|4000000|4000000000|494640000000|2.12765957446809|46|046||. -0|3|300|3000000|3000000000|494726400000|2.08333333333333|47|047||/ -1|2|200|2000000|2000000000|494812800000|2.04081632653061|48|048||0 -0|1|100|1000000|1000000000|494899200000|2.0|49|049||1 -1|0|0|0|0|494985600000|1.96078431372549|50|050||2 -0|-1|-100|-1000000|-1000000000|495072000000|1.92307692307692|51|051||3 -1|-2|-200|-2000000|-2000000000|495158400000|1.88679245283019|52|052||4 -0|-3|-300|-3000000|-3000000000|495244800000|1.85185185185185|53|053||5 -1|-4|-400|-4000000|-4000000000|495331200000|1.81818181818182|54|054||6 -0|-5|-500|-5000000|-5000000000|495417600000|1.78571428571429|55|055||7 -1|-6|-600|-6000000|-6000000000|495504000000|1.75438596491228|56|056||8 -0|-7|-700|-7000000|-7000000000|495590400000|1.72413793103448|57|057||9 -1|-8|-800|-8000000|-8000000000|495676800000|1.69491525423729|58|058||: -0|-9|-900|-9000000|-9000000000|495763200000|1.66666666666667|59|059||; -1|-10|-1000|-10000000|-10000000000|495849600000|1.63934426229508|60|060||< -0|-11|-1100|-11000000|-11000000000|495936000000|1.61290322580645|61|061||= -1|-12|-1200|-12000000|-12000000000|496022400000|1.58730158730159|62|062||> -0|-13|-1300|-13000000|-13000000000|496108800000|1.5625|63|063||? -1|-14|-1400|-14000000|-14000000000|496195200000|1.53846153846154|64|064||@ -0|-15|-1500|-15000000|-15000000000|496281600000|1.51515151515152|65|065||A -1|-16|-1600|-16000000|-16000000000|496368000000|1.49253731343284|66|066||B -0|-17|-1700|-17000000|-17000000000|496454400000|1.47058823529412|67|067||C -1|-18|-1800|-18000000|-18000000000|496540800000|1.44927536231884|68|068||D -0|-19|-1900|-19000000|-19000000000|496627200000|1.42857142857143|69|069||E -1|-20|-2000|-20000000|-20000000000|496713600000|1.40845070422535|70|070||F -0|-21|-2100|-21000000|-21000000000|496800000000|1.38888888888889|71|071||G -1|-22|-2200|-22000000|-22000000000|496886400000|1.36986301369863|72|072||H -0|-23|-2300|-23000000|-23000000000|496972800000|1.35135135135135|73|073||I -1|-24|-2400|-24000000|-24000000000|497059200000|1.33333333333333|74|074||J -0|-25|-2500|-25000000|-25000000000|497145600000|1.31578947368421|75|075||K -1|-26|-2600|-26000000|-26000000000|497232000000|1.2987012987013|76|076||L -0|-27|-2700|-27000000|-27000000000|497318400000|1.28205128205128|77|077||M -1|-28|-2800|-28000000|-28000000000|497404800000|1.26582278481013|78|078||N -0|-29|-2900|-29000000|-29000000000|497491200000|1.25|79|079||O -1|-30|-3000|-30000000|-30000000000|497577600000|1.23456790123457|80|080||P -0|-31|-3100|-31000000|-31000000000|497664000000|1.21951219512195|81|081||Q -1|-32|-3200|-32000000|-32000000000|497750400000|1.20481927710843|82|082||R -0|-33|-3300|-33000000|-33000000000|497836800000|1.19047619047619|83|083||S -1|-34|-3400|-34000000|-34000000000|497923200000|1.17647058823529|84|084||T -0|-35|-3500|-35000000|-35000000000|498009600000|1.16279069767442|85|085||U -1|-36|-3600|-36000000|-36000000000|498096000000|1.14942528735632|86|086||V -0|-37|-3700|-37000000|-37000000000|498182400000|1.13636363636364|87|087||W -1|-38|-3800|-38000000|-38000000000|498268800000|1.12359550561798|88|088||X -0|-39|-3900|-39000000|-39000000000|498355200000|1.11111111111111|89|089||Y -1|-40|-4000|-40000000|-40000000000|498441600000|1.0989010989011|90|090||Z -0|-41|-4100|-41000000|-41000000000|498528000000|1.08695652173913|91|091||[ -1|-42|-4200|-42000000|-42000000000|498614400000|1.0752688172043|92|092||\ -0|-43|-4300|-43000000|-43000000000|498700800000|1.06382978723404|93|093||] -1|-44|-4400|-44000000|-44000000000|498787200000|1.05263157894737|94|094||^ -0|-45|-4500|-45000000|-45000000000|498873600000|1.04166666666667|95|095||_ -1|-46|-4600|-46000000|-46000000000|498960000000|1.03092783505155|96|096||` -0|-47|-4700|-47000000|-47000000000|499046400000|1.02040816326531|97|097||a -1|-48|-4800|-48000000|-48000000000|499132800000|1.01010101010101|98|098||b -0|-49|-4900|-49000000|-49000000000|499219200000|1.0|99|099||c +SELECT rowid, bool_0, int8_1, int16_2, int32_3, int64_4, datetime(ts_5 / 1000, 'unixepoch'), printf("%.4f", double_6) AS double_6, string_7, string_8, quote(binary_9), quote(binary_10) FROM test; +0|1|50|5000|50000000|50000000000|1985-07-20 00:00:00|100.0000|0|000|X'00'|X'00' +1|0|49|4900|49000000|49000000000|1985-07-21 00:00:00|50.0000|1|001|X'0101'|X'01' +2|1|48|4800|48000000|48000000000|1985-07-22 00:00:00|33.3333|2|002|X'020202'|X'02' +3|0|47|4700|47000000|47000000000|1985-07-23 00:00:00|25.0000|3|003|X'03030303'|X'03' +4|1|46|4600|46000000|46000000000|1985-07-24 00:00:00|20.0000|4|004|X'0404040404'|X'04' +5|0|45|4500|45000000|45000000000|1985-07-25 00:00:00|16.6667|5|005|X'05'|X'05' +6|1|44|4400|44000000|44000000000|1985-07-26 00:00:00|14.2857|6|006|X'0606'|X'06' +7|0|43|4300|43000000|43000000000|1985-07-27 00:00:00|12.5000|7|007|X'070707'|X'07' +8|1|42|4200|42000000|42000000000|1985-07-28 00:00:00|11.1111|8|008|X'08080808'|X'08' +9|0|41|4100|41000000|41000000000|1985-07-29 00:00:00|10.0000|9|009|X'0909090909'|X'09' +10|1|40|4000|40000000|40000000000|1985-07-30 00:00:00|9.0909|10|010|X'0A'|X'0A' +11|0|39|3900|39000000|39000000000|1985-07-31 00:00:00|8.3333|11|011|X'0B0B'|X'0B' +12|1|38|3800|38000000|38000000000|1985-08-01 00:00:00|7.6923|12|012|X'0C0C0C'|X'0C' +13|0|37|3700|37000000|37000000000|1985-08-02 00:00:00|7.1429|13|013|X'0D0D0D0D'|X'0D' +14|1|36|3600|36000000|36000000000|1985-08-03 00:00:00|6.6667|14|014|X'0E0E0E0E0E'|X'0E' +15|0|35|3500|35000000|35000000000|1985-08-04 00:00:00|6.2500|15|015|X'0F'|X'0F' +16|1|34|3400|34000000|34000000000|1985-08-05 00:00:00|5.8824|16|016|X'1010'|X'10' +17|0|33|3300|33000000|33000000000|1985-08-06 00:00:00|5.5556|17|017|X'111111'|X'11' +18|1|32|3200|32000000|32000000000|1985-08-07 00:00:00|5.2632|18|018|X'12121212'|X'12' +19|0|31|3100|31000000|31000000000|1985-08-08 00:00:00|5.0000|19|019|X'1313131313'|X'13' +20|1|30|3000|30000000|30000000000|1985-08-09 00:00:00|4.7619|20|020|X'14'|X'14' +21|0|29|2900|29000000|29000000000|1985-08-10 00:00:00|4.5455|21|021|X'1515'|X'15' +22|1|28|2800|28000000|28000000000|1985-08-11 00:00:00|4.3478|22|022|X'161616'|X'16' +23|0|27|2700|27000000|27000000000|1985-08-12 00:00:00|4.1667|23|023|X'17171717'|X'17' +24|1|26|2600|26000000|26000000000|1985-08-13 00:00:00|4.0000|24|024|X'1818181818'|X'18' +25|0|25|2500|25000000|25000000000|1985-08-14 00:00:00|3.8462|25|025|X'19'|X'19' +26|1|24|2400|24000000|24000000000|1985-08-15 00:00:00|3.7037|26|026|X'1A1A'|X'1A' +27|0|23|2300|23000000|23000000000|1985-08-16 00:00:00|3.5714|27|027|X'1B1B1B'|X'1B' +28|1|22|2200|22000000|22000000000|1985-08-17 00:00:00|3.4483|28|028|X'1C1C1C1C'|X'1C' +29|0|21|2100|21000000|21000000000|1985-08-18 00:00:00|3.3333|29|029|X'1D1D1D1D1D'|X'1D' +30|1|20|2000|20000000|20000000000|1985-08-19 00:00:00|3.2258|30|030|X'1E'|X'1E' +31|0|19|1900|19000000|19000000000|1985-08-20 00:00:00|3.1250|31|031|X'1F1F'|X'1F' +32|1|18|1800|18000000|18000000000|1985-08-21 00:00:00|3.0303|32|032|X'202020'|X'20' +33|0|17|1700|17000000|17000000000|1985-08-22 00:00:00|2.9412|33|033|X'21212121'|X'21' +34|1|16|1600|16000000|16000000000|1985-08-23 00:00:00|2.8571|34|034|X'2222222222'|X'22' +35|0|15|1500|15000000|15000000000|1985-08-24 00:00:00|2.7778|35|035|X'23'|X'23' +36|1|14|1400|14000000|14000000000|1985-08-25 00:00:00|2.7027|36|036|X'2424'|X'24' +37|0|13|1300|13000000|13000000000|1985-08-26 00:00:00|2.6316|37|037|X'252525'|X'25' +38|1|12|1200|12000000|12000000000|1985-08-27 00:00:00|2.5641|38|038|X'26262626'|X'26' +39|0|11|1100|11000000|11000000000|1985-08-28 00:00:00|2.5000|39|039|X'2727272727'|X'27' +40|1|10|1000|10000000|10000000000|1985-08-29 00:00:00|2.4390|40|040|X'28'|X'28' +41|0|9|900|9000000|9000000000|1985-08-30 00:00:00|2.3810|41|041|X'2929'|X'29' +42|1|8|800|8000000|8000000000|1985-08-31 00:00:00|2.3256|42|042|X'2A2A2A'|X'2A' +43|0|7|700|7000000|7000000000|1985-09-01 00:00:00|2.2727|43|043|X'2B2B2B2B'|X'2B' +44|1|6|600|6000000|6000000000|1985-09-02 00:00:00|2.2222|44|044|X'2C2C2C2C2C'|X'2C' +45|0|5|500|5000000|5000000000|1985-09-03 00:00:00|2.1739|45|045|X'2D'|X'2D' +46|1|4|400|4000000|4000000000|1985-09-04 00:00:00|2.1277|46|046|X'2E2E'|X'2E' +47|0|3|300|3000000|3000000000|1985-09-05 00:00:00|2.0833|47|047|X'2F2F2F'|X'2F' +48|1|2|200|2000000|2000000000|1985-09-06 00:00:00|2.0408|48|048|X'30303030'|X'30' +49|0|1|100|1000000|1000000000|1985-09-07 00:00:00|2.0000|49|049|X'3131313131'|X'31' +50|1|0|0|0|0|1985-09-08 00:00:00|1.9608|50|050|X'32'|X'32' +51|0|-1|-100|-1000000|-1000000000|1985-09-09 00:00:00|1.9231|51|051|X'3333'|X'33' +52|1|-2|-200|-2000000|-2000000000|1985-09-10 00:00:00|1.8868|52|052|X'343434'|X'34' +53|0|-3|-300|-3000000|-3000000000|1985-09-11 00:00:00|1.8519|53|053|X'35353535'|X'35' +54|1|-4|-400|-4000000|-4000000000|1985-09-12 00:00:00|1.8182|54|054|X'3636363636'|X'36' +55|0|-5|-500|-5000000|-5000000000|1985-09-13 00:00:00|1.7857|55|055|X'37'|X'37' +56|1|-6|-600|-6000000|-6000000000|1985-09-14 00:00:00|1.7544|56|056|X'3838'|X'38' +57|0|-7|-700|-7000000|-7000000000|1985-09-15 00:00:00|1.7241|57|057|X'393939'|X'39' +58|1|-8|-800|-8000000|-8000000000|1985-09-16 00:00:00|1.6949|58|058|X'3A3A3A3A'|X'3A' +59|0|-9|-900|-9000000|-9000000000|1985-09-17 00:00:00|1.6667|59|059|X'3B3B3B3B3B'|X'3B' +60|1|-10|-1000|-10000000|-10000000000|1985-09-18 00:00:00|1.6393|60|060|X'3C'|X'3C' +61|0|-11|-1100|-11000000|-11000000000|1985-09-19 00:00:00|1.6129|61|061|X'3D3D'|X'3D' +62|1|-12|-1200|-12000000|-12000000000|1985-09-20 00:00:00|1.5873|62|062|X'3E3E3E'|X'3E' +63|0|-13|-1300|-13000000|-13000000000|1985-09-21 00:00:00|1.5625|63|063|X'3F3F3F3F'|X'3F' +64|1|-14|-1400|-14000000|-14000000000|1985-09-22 00:00:00|1.5385|64|064|X'4040404040'|X'40' +65|0|-15|-1500|-15000000|-15000000000|1985-09-23 00:00:00|1.5152|65|065|X'41'|X'41' +66|1|-16|-1600|-16000000|-16000000000|1985-09-24 00:00:00|1.4925|66|066|X'4242'|X'42' +67|0|-17|-1700|-17000000|-17000000000|1985-09-25 00:00:00|1.4706|67|067|X'434343'|X'43' +68|1|-18|-1800|-18000000|-18000000000|1985-09-26 00:00:00|1.4493|68|068|X'44444444'|X'44' +69|0|-19|-1900|-19000000|-19000000000|1985-09-27 00:00:00|1.4286|69|069|X'4545454545'|X'45' +70|1|-20|-2000|-20000000|-20000000000|1985-09-28 00:00:00|1.4085|70|070|X'46'|X'46' +71|0|-21|-2100|-21000000|-21000000000|1985-09-29 00:00:00|1.3889|71|071|X'4747'|X'47' +72|1|-22|-2200|-22000000|-22000000000|1985-09-30 00:00:00|1.3699|72|072|X'484848'|X'48' +73|0|-23|-2300|-23000000|-23000000000|1985-10-01 00:00:00|1.3514|73|073|X'49494949'|X'49' +74|1|-24|-2400|-24000000|-24000000000|1985-10-02 00:00:00|1.3333|74|074|X'4A4A4A4A4A'|X'4A' +75|0|-25|-2500|-25000000|-25000000000|1985-10-03 00:00:00|1.3158|75|075|X'4B'|X'4B' +76|1|-26|-2600|-26000000|-26000000000|1985-10-04 00:00:00|1.2987|76|076|X'4C4C'|X'4C' +77|0|-27|-2700|-27000000|-27000000000|1985-10-05 00:00:00|1.2821|77|077|X'4D4D4D'|X'4D' +78|1|-28|-2800|-28000000|-28000000000|1985-10-06 00:00:00|1.2658|78|078|X'4E4E4E4E'|X'4E' +79|0|-29|-2900|-29000000|-29000000000|1985-10-07 00:00:00|1.2500|79|079|X'4F4F4F4F4F'|X'4F' +80|1|-30|-3000|-30000000|-30000000000|1985-10-08 00:00:00|1.2346|80|080|X'50'|X'50' +81|0|-31|-3100|-31000000|-31000000000|1985-10-09 00:00:00|1.2195|81|081|X'5151'|X'51' +82|1|-32|-3200|-32000000|-32000000000|1985-10-10 00:00:00|1.2048|82|082|X'525252'|X'52' +83|0|-33|-3300|-33000000|-33000000000|1985-10-11 00:00:00|1.1905|83|083|X'53535353'|X'53' +84|1|-34|-3400|-34000000|-34000000000|1985-10-12 00:00:00|1.1765|84|084|X'5454545454'|X'54' +85|0|-35|-3500|-35000000|-35000000000|1985-10-13 00:00:00|1.1628|85|085|X'55'|X'55' +86|1|-36|-3600|-36000000|-36000000000|1985-10-14 00:00:00|1.1494|86|086|X'5656'|X'56' +87|0|-37|-3700|-37000000|-37000000000|1985-10-15 00:00:00|1.1364|87|087|X'575757'|X'57' +88|1|-38|-3800|-38000000|-38000000000|1985-10-16 00:00:00|1.1236|88|088|X'58585858'|X'58' +89|0|-39|-3900|-39000000|-39000000000|1985-10-17 00:00:00|1.1111|89|089|X'5959595959'|X'59' +90|1|-40|-4000|-40000000|-40000000000|1985-10-18 00:00:00|1.0989|90|090|X'5A'|X'5A' +91|0|-41|-4100|-41000000|-41000000000|1985-10-19 00:00:00|1.0870|91|091|X'5B5B'|X'5B' +92|1|-42|-4200|-42000000|-42000000000|1985-10-20 00:00:00|1.0753|92|092|X'5C5C5C'|X'5C' +93|0|-43|-4300|-43000000|-43000000000|1985-10-21 00:00:00|1.0638|93|093|X'5D5D5D5D'|X'5D' +94|1|-44|-4400|-44000000|-44000000000|1985-10-22 00:00:00|1.0526|94|094|X'5E5E5E5E5E'|X'5E' +95|0|-45|-4500|-45000000|-45000000000|1985-10-23 00:00:00|1.0417|95|095|X'5F'|X'5F' +96|1|-46|-4600|-46000000|-46000000000|1985-10-24 00:00:00|1.0309|96|096|X'6060'|X'60' +97|0|-47|-4700|-47000000|-47000000000|1985-10-25 00:00:00|1.0204|97|097|X'616161'|X'61' +98|1|-48|-4800|-48000000|-48000000000|1985-10-26 00:00:00|1.0101|98|098|X'62626262'|X'62' +99|0|-49|-4900|-49000000|-49000000000|1985-10-27 00:00:00|1.0000|99|099|X'6363636363'|X'63' diff --git a/tests/queries/002-table-scan-10-rowgroups.sql b/tests/queries/002-table-scan-10-rowgroups.sql index 5061a30..0793844 100644 --- a/tests/queries/002-table-scan-10-rowgroups.sql +++ b/tests/queries/002-table-scan-10-rowgroups.sql @@ -1,103 +1,102 @@ 100-rows-10.parquet -select * from test; -1|50|5000|50000000|50000000000|490665600000|100.0|0|000|| -0|49|4900|49000000|49000000000|490752000000|50.0|1|001|| -1|48|4800|48000000|48000000000|490838400000|33.3333333333333|2|002|| -0|47|4700|47000000|47000000000|490924800000|25.0|3|003|| -1|46|4600|46000000|46000000000|491011200000|20.0|4|004|| -0|45|4500|45000000|45000000000|491097600000|16.6666666666667|5|005|| -1|44|4400|44000000|44000000000|491184000000|14.2857142857143|6|006|| -0|43|4300|43000000|43000000000|491270400000|12.5|7|007|| -1|42|4200|42000000|42000000000|491356800000|11.1111111111111|8|008|| -0|41|4100|41000000|41000000000|491443200000|10.0|9|009|| -1|40|4000|40000000|40000000000|491529600000|9.09090909090909|10|010|| - -0|39|3900|39000000|39000000000|491616000000|8.33333333333333|11|011|| -1|38|3800|38000000|38000000000|491702400000|7.69230769230769|12|012|| -0|37|3700|37000000|37000000000|491788800000|7.14285714285714|13|013|| -1|36|3600|36000000|36000000000|491875200000|6.66666666666667|14|014|| -0|35|3500|35000000|35000000000|491961600000|6.25|15|015|| -1|34|3400|34000000|34000000000|492048000000|5.88235294117647|16|016|| -0|33|3300|33000000|33000000000|492134400000|5.55555555555556|17|017|| -1|32|3200|32000000|32000000000|492220800000|5.26315789473684|18|018|| -0|31|3100|31000000|31000000000|492307200000|5.0|19|019|| -1|30|3000|30000000|30000000000|492393600000|4.76190476190476|20|020|| -0|29|2900|29000000|29000000000|492480000000|4.54545454545455|21|021|| -1|28|2800|28000000|28000000000|492566400000|4.34782608695652|22|022|| -0|27|2700|27000000|27000000000|492652800000|4.16666666666667|23|023|| -1|26|2600|26000000|26000000000|492739200000|4.0|24|024|| -0|25|2500|25000000|25000000000|492825600000|3.84615384615385|25|025|| -1|24|2400|24000000|24000000000|492912000000|3.7037037037037|26|026|| -0|23|2300|23000000|23000000000|492998400000|3.57142857142857|27|027|| -1|22|2200|22000000|22000000000|493084800000|3.44827586206897|28|028|| -0|21|2100|21000000|21000000000|493171200000|3.33333333333333|29|029|| -1|20|2000|20000000|20000000000|493257600000|3.2258064516129|30|030|| -0|19|1900|19000000|19000000000|493344000000|3.125|31|031|| -1|18|1800|18000000|18000000000|493430400000|3.03030303030303|32|032|| -0|17|1700|17000000|17000000000|493516800000|2.94117647058824|33|033||! -1|16|1600|16000000|16000000000|493603200000|2.85714285714286|34|034||" -0|15|1500|15000000|15000000000|493689600000|2.77777777777778|35|035||# -1|14|1400|14000000|14000000000|493776000000|2.7027027027027|36|036||$ -0|13|1300|13000000|13000000000|493862400000|2.63157894736842|37|037||% -1|12|1200|12000000|12000000000|493948800000|2.56410256410256|38|038||& -0|11|1100|11000000|11000000000|494035200000|2.5|39|039||' -1|10|1000|10000000|10000000000|494121600000|2.4390243902439|40|040||( -0|9|900|9000000|9000000000|494208000000|2.38095238095238|41|041||) -1|8|800|8000000|8000000000|494294400000|2.32558139534884|42|042||* -0|7|700|7000000|7000000000|494380800000|2.27272727272727|43|043||+ -1|6|600|6000000|6000000000|494467200000|2.22222222222222|44|044||, -0|5|500|5000000|5000000000|494553600000|2.17391304347826|45|045||- -1|4|400|4000000|4000000000|494640000000|2.12765957446809|46|046||. -0|3|300|3000000|3000000000|494726400000|2.08333333333333|47|047||/ -1|2|200|2000000|2000000000|494812800000|2.04081632653061|48|048||0 -0|1|100|1000000|1000000000|494899200000|2.0|49|049||1 -1|0|0|0|0|494985600000|1.96078431372549|50|050||2 -0|-1|-100|-1000000|-1000000000|495072000000|1.92307692307692|51|051||3 -1|-2|-200|-2000000|-2000000000|495158400000|1.88679245283019|52|052||4 -0|-3|-300|-3000000|-3000000000|495244800000|1.85185185185185|53|053||5 -1|-4|-400|-4000000|-4000000000|495331200000|1.81818181818182|54|054||6 -0|-5|-500|-5000000|-5000000000|495417600000|1.78571428571429|55|055||7 -1|-6|-600|-6000000|-6000000000|495504000000|1.75438596491228|56|056||8 -0|-7|-700|-7000000|-7000000000|495590400000|1.72413793103448|57|057||9 -1|-8|-800|-8000000|-8000000000|495676800000|1.69491525423729|58|058||: -0|-9|-900|-9000000|-9000000000|495763200000|1.66666666666667|59|059||; -1|-10|-1000|-10000000|-10000000000|495849600000|1.63934426229508|60|060||< -0|-11|-1100|-11000000|-11000000000|495936000000|1.61290322580645|61|061||= -1|-12|-1200|-12000000|-12000000000|496022400000|1.58730158730159|62|062||> -0|-13|-1300|-13000000|-13000000000|496108800000|1.5625|63|063||? -1|-14|-1400|-14000000|-14000000000|496195200000|1.53846153846154|64|064||@ -0|-15|-1500|-15000000|-15000000000|496281600000|1.51515151515152|65|065||A -1|-16|-1600|-16000000|-16000000000|496368000000|1.49253731343284|66|066||B -0|-17|-1700|-17000000|-17000000000|496454400000|1.47058823529412|67|067||C -1|-18|-1800|-18000000|-18000000000|496540800000|1.44927536231884|68|068||D -0|-19|-1900|-19000000|-19000000000|496627200000|1.42857142857143|69|069||E -1|-20|-2000|-20000000|-20000000000|496713600000|1.40845070422535|70|070||F -0|-21|-2100|-21000000|-21000000000|496800000000|1.38888888888889|71|071||G -1|-22|-2200|-22000000|-22000000000|496886400000|1.36986301369863|72|072||H -0|-23|-2300|-23000000|-23000000000|496972800000|1.35135135135135|73|073||I -1|-24|-2400|-24000000|-24000000000|497059200000|1.33333333333333|74|074||J -0|-25|-2500|-25000000|-25000000000|497145600000|1.31578947368421|75|075||K -1|-26|-2600|-26000000|-26000000000|497232000000|1.2987012987013|76|076||L -0|-27|-2700|-27000000|-27000000000|497318400000|1.28205128205128|77|077||M -1|-28|-2800|-28000000|-28000000000|497404800000|1.26582278481013|78|078||N -0|-29|-2900|-29000000|-29000000000|497491200000|1.25|79|079||O -1|-30|-3000|-30000000|-30000000000|497577600000|1.23456790123457|80|080||P -0|-31|-3100|-31000000|-31000000000|497664000000|1.21951219512195|81|081||Q -1|-32|-3200|-32000000|-32000000000|497750400000|1.20481927710843|82|082||R -0|-33|-3300|-33000000|-33000000000|497836800000|1.19047619047619|83|083||S -1|-34|-3400|-34000000|-34000000000|497923200000|1.17647058823529|84|084||T -0|-35|-3500|-35000000|-35000000000|498009600000|1.16279069767442|85|085||U -1|-36|-3600|-36000000|-36000000000|498096000000|1.14942528735632|86|086||V -0|-37|-3700|-37000000|-37000000000|498182400000|1.13636363636364|87|087||W -1|-38|-3800|-38000000|-38000000000|498268800000|1.12359550561798|88|088||X -0|-39|-3900|-39000000|-39000000000|498355200000|1.11111111111111|89|089||Y -1|-40|-4000|-40000000|-40000000000|498441600000|1.0989010989011|90|090||Z -0|-41|-4100|-41000000|-41000000000|498528000000|1.08695652173913|91|091||[ -1|-42|-4200|-42000000|-42000000000|498614400000|1.0752688172043|92|092||\ -0|-43|-4300|-43000000|-43000000000|498700800000|1.06382978723404|93|093||] -1|-44|-4400|-44000000|-44000000000|498787200000|1.05263157894737|94|094||^ -0|-45|-4500|-45000000|-45000000000|498873600000|1.04166666666667|95|095||_ -1|-46|-4600|-46000000|-46000000000|498960000000|1.03092783505155|96|096||` -0|-47|-4700|-47000000|-47000000000|499046400000|1.02040816326531|97|097||a -1|-48|-4800|-48000000|-48000000000|499132800000|1.01010101010101|98|098||b -0|-49|-4900|-49000000|-49000000000|499219200000|1.0|99|099||c +SELECT rowid, bool_0, int8_1, int16_2, int32_3, int64_4, datetime(ts_5 / 1000, 'unixepoch'), printf("%.4f", double_6) AS double_6, string_7, string_8, quote(binary_9), quote(binary_10) FROM test; +0|1|50|5000|50000000|50000000000|1985-07-20 00:00:00|100.0000|0|000|X'00'|X'00' +1|0|49|4900|49000000|49000000000|1985-07-21 00:00:00|50.0000|1|001|X'0101'|X'01' +2|1|48|4800|48000000|48000000000|1985-07-22 00:00:00|33.3333|2|002|X'020202'|X'02' +3|0|47|4700|47000000|47000000000|1985-07-23 00:00:00|25.0000|3|003|X'03030303'|X'03' +4|1|46|4600|46000000|46000000000|1985-07-24 00:00:00|20.0000|4|004|X'0404040404'|X'04' +5|0|45|4500|45000000|45000000000|1985-07-25 00:00:00|16.6667|5|005|X'05'|X'05' +6|1|44|4400|44000000|44000000000|1985-07-26 00:00:00|14.2857|6|006|X'0606'|X'06' +7|0|43|4300|43000000|43000000000|1985-07-27 00:00:00|12.5000|7|007|X'070707'|X'07' +8|1|42|4200|42000000|42000000000|1985-07-28 00:00:00|11.1111|8|008|X'08080808'|X'08' +9|0|41|4100|41000000|41000000000|1985-07-29 00:00:00|10.0000|9|009|X'0909090909'|X'09' +10|1|40|4000|40000000|40000000000|1985-07-30 00:00:00|9.0909|10|010|X'0A'|X'0A' +11|0|39|3900|39000000|39000000000|1985-07-31 00:00:00|8.3333|11|011|X'0B0B'|X'0B' +12|1|38|3800|38000000|38000000000|1985-08-01 00:00:00|7.6923|12|012|X'0C0C0C'|X'0C' +13|0|37|3700|37000000|37000000000|1985-08-02 00:00:00|7.1429|13|013|X'0D0D0D0D'|X'0D' +14|1|36|3600|36000000|36000000000|1985-08-03 00:00:00|6.6667|14|014|X'0E0E0E0E0E'|X'0E' +15|0|35|3500|35000000|35000000000|1985-08-04 00:00:00|6.2500|15|015|X'0F'|X'0F' +16|1|34|3400|34000000|34000000000|1985-08-05 00:00:00|5.8824|16|016|X'1010'|X'10' +17|0|33|3300|33000000|33000000000|1985-08-06 00:00:00|5.5556|17|017|X'111111'|X'11' +18|1|32|3200|32000000|32000000000|1985-08-07 00:00:00|5.2632|18|018|X'12121212'|X'12' +19|0|31|3100|31000000|31000000000|1985-08-08 00:00:00|5.0000|19|019|X'1313131313'|X'13' +20|1|30|3000|30000000|30000000000|1985-08-09 00:00:00|4.7619|20|020|X'14'|X'14' +21|0|29|2900|29000000|29000000000|1985-08-10 00:00:00|4.5455|21|021|X'1515'|X'15' +22|1|28|2800|28000000|28000000000|1985-08-11 00:00:00|4.3478|22|022|X'161616'|X'16' +23|0|27|2700|27000000|27000000000|1985-08-12 00:00:00|4.1667|23|023|X'17171717'|X'17' +24|1|26|2600|26000000|26000000000|1985-08-13 00:00:00|4.0000|24|024|X'1818181818'|X'18' +25|0|25|2500|25000000|25000000000|1985-08-14 00:00:00|3.8462|25|025|X'19'|X'19' +26|1|24|2400|24000000|24000000000|1985-08-15 00:00:00|3.7037|26|026|X'1A1A'|X'1A' +27|0|23|2300|23000000|23000000000|1985-08-16 00:00:00|3.5714|27|027|X'1B1B1B'|X'1B' +28|1|22|2200|22000000|22000000000|1985-08-17 00:00:00|3.4483|28|028|X'1C1C1C1C'|X'1C' +29|0|21|2100|21000000|21000000000|1985-08-18 00:00:00|3.3333|29|029|X'1D1D1D1D1D'|X'1D' +30|1|20|2000|20000000|20000000000|1985-08-19 00:00:00|3.2258|30|030|X'1E'|X'1E' +31|0|19|1900|19000000|19000000000|1985-08-20 00:00:00|3.1250|31|031|X'1F1F'|X'1F' +32|1|18|1800|18000000|18000000000|1985-08-21 00:00:00|3.0303|32|032|X'202020'|X'20' +33|0|17|1700|17000000|17000000000|1985-08-22 00:00:00|2.9412|33|033|X'21212121'|X'21' +34|1|16|1600|16000000|16000000000|1985-08-23 00:00:00|2.8571|34|034|X'2222222222'|X'22' +35|0|15|1500|15000000|15000000000|1985-08-24 00:00:00|2.7778|35|035|X'23'|X'23' +36|1|14|1400|14000000|14000000000|1985-08-25 00:00:00|2.7027|36|036|X'2424'|X'24' +37|0|13|1300|13000000|13000000000|1985-08-26 00:00:00|2.6316|37|037|X'252525'|X'25' +38|1|12|1200|12000000|12000000000|1985-08-27 00:00:00|2.5641|38|038|X'26262626'|X'26' +39|0|11|1100|11000000|11000000000|1985-08-28 00:00:00|2.5000|39|039|X'2727272727'|X'27' +40|1|10|1000|10000000|10000000000|1985-08-29 00:00:00|2.4390|40|040|X'28'|X'28' +41|0|9|900|9000000|9000000000|1985-08-30 00:00:00|2.3810|41|041|X'2929'|X'29' +42|1|8|800|8000000|8000000000|1985-08-31 00:00:00|2.3256|42|042|X'2A2A2A'|X'2A' +43|0|7|700|7000000|7000000000|1985-09-01 00:00:00|2.2727|43|043|X'2B2B2B2B'|X'2B' +44|1|6|600|6000000|6000000000|1985-09-02 00:00:00|2.2222|44|044|X'2C2C2C2C2C'|X'2C' +45|0|5|500|5000000|5000000000|1985-09-03 00:00:00|2.1739|45|045|X'2D'|X'2D' +46|1|4|400|4000000|4000000000|1985-09-04 00:00:00|2.1277|46|046|X'2E2E'|X'2E' +47|0|3|300|3000000|3000000000|1985-09-05 00:00:00|2.0833|47|047|X'2F2F2F'|X'2F' +48|1|2|200|2000000|2000000000|1985-09-06 00:00:00|2.0408|48|048|X'30303030'|X'30' +49|0|1|100|1000000|1000000000|1985-09-07 00:00:00|2.0000|49|049|X'3131313131'|X'31' +50|1|0|0|0|0|1985-09-08 00:00:00|1.9608|50|050|X'32'|X'32' +51|0|-1|-100|-1000000|-1000000000|1985-09-09 00:00:00|1.9231|51|051|X'3333'|X'33' +52|1|-2|-200|-2000000|-2000000000|1985-09-10 00:00:00|1.8868|52|052|X'343434'|X'34' +53|0|-3|-300|-3000000|-3000000000|1985-09-11 00:00:00|1.8519|53|053|X'35353535'|X'35' +54|1|-4|-400|-4000000|-4000000000|1985-09-12 00:00:00|1.8182|54|054|X'3636363636'|X'36' +55|0|-5|-500|-5000000|-5000000000|1985-09-13 00:00:00|1.7857|55|055|X'37'|X'37' +56|1|-6|-600|-6000000|-6000000000|1985-09-14 00:00:00|1.7544|56|056|X'3838'|X'38' +57|0|-7|-700|-7000000|-7000000000|1985-09-15 00:00:00|1.7241|57|057|X'393939'|X'39' +58|1|-8|-800|-8000000|-8000000000|1985-09-16 00:00:00|1.6949|58|058|X'3A3A3A3A'|X'3A' +59|0|-9|-900|-9000000|-9000000000|1985-09-17 00:00:00|1.6667|59|059|X'3B3B3B3B3B'|X'3B' +60|1|-10|-1000|-10000000|-10000000000|1985-09-18 00:00:00|1.6393|60|060|X'3C'|X'3C' +61|0|-11|-1100|-11000000|-11000000000|1985-09-19 00:00:00|1.6129|61|061|X'3D3D'|X'3D' +62|1|-12|-1200|-12000000|-12000000000|1985-09-20 00:00:00|1.5873|62|062|X'3E3E3E'|X'3E' +63|0|-13|-1300|-13000000|-13000000000|1985-09-21 00:00:00|1.5625|63|063|X'3F3F3F3F'|X'3F' +64|1|-14|-1400|-14000000|-14000000000|1985-09-22 00:00:00|1.5385|64|064|X'4040404040'|X'40' +65|0|-15|-1500|-15000000|-15000000000|1985-09-23 00:00:00|1.5152|65|065|X'41'|X'41' +66|1|-16|-1600|-16000000|-16000000000|1985-09-24 00:00:00|1.4925|66|066|X'4242'|X'42' +67|0|-17|-1700|-17000000|-17000000000|1985-09-25 00:00:00|1.4706|67|067|X'434343'|X'43' +68|1|-18|-1800|-18000000|-18000000000|1985-09-26 00:00:00|1.4493|68|068|X'44444444'|X'44' +69|0|-19|-1900|-19000000|-19000000000|1985-09-27 00:00:00|1.4286|69|069|X'4545454545'|X'45' +70|1|-20|-2000|-20000000|-20000000000|1985-09-28 00:00:00|1.4085|70|070|X'46'|X'46' +71|0|-21|-2100|-21000000|-21000000000|1985-09-29 00:00:00|1.3889|71|071|X'4747'|X'47' +72|1|-22|-2200|-22000000|-22000000000|1985-09-30 00:00:00|1.3699|72|072|X'484848'|X'48' +73|0|-23|-2300|-23000000|-23000000000|1985-10-01 00:00:00|1.3514|73|073|X'49494949'|X'49' +74|1|-24|-2400|-24000000|-24000000000|1985-10-02 00:00:00|1.3333|74|074|X'4A4A4A4A4A'|X'4A' +75|0|-25|-2500|-25000000|-25000000000|1985-10-03 00:00:00|1.3158|75|075|X'4B'|X'4B' +76|1|-26|-2600|-26000000|-26000000000|1985-10-04 00:00:00|1.2987|76|076|X'4C4C'|X'4C' +77|0|-27|-2700|-27000000|-27000000000|1985-10-05 00:00:00|1.2821|77|077|X'4D4D4D'|X'4D' +78|1|-28|-2800|-28000000|-28000000000|1985-10-06 00:00:00|1.2658|78|078|X'4E4E4E4E'|X'4E' +79|0|-29|-2900|-29000000|-29000000000|1985-10-07 00:00:00|1.2500|79|079|X'4F4F4F4F4F'|X'4F' +80|1|-30|-3000|-30000000|-30000000000|1985-10-08 00:00:00|1.2346|80|080|X'50'|X'50' +81|0|-31|-3100|-31000000|-31000000000|1985-10-09 00:00:00|1.2195|81|081|X'5151'|X'51' +82|1|-32|-3200|-32000000|-32000000000|1985-10-10 00:00:00|1.2048|82|082|X'525252'|X'52' +83|0|-33|-3300|-33000000|-33000000000|1985-10-11 00:00:00|1.1905|83|083|X'53535353'|X'53' +84|1|-34|-3400|-34000000|-34000000000|1985-10-12 00:00:00|1.1765|84|084|X'5454545454'|X'54' +85|0|-35|-3500|-35000000|-35000000000|1985-10-13 00:00:00|1.1628|85|085|X'55'|X'55' +86|1|-36|-3600|-36000000|-36000000000|1985-10-14 00:00:00|1.1494|86|086|X'5656'|X'56' +87|0|-37|-3700|-37000000|-37000000000|1985-10-15 00:00:00|1.1364|87|087|X'575757'|X'57' +88|1|-38|-3800|-38000000|-38000000000|1985-10-16 00:00:00|1.1236|88|088|X'58585858'|X'58' +89|0|-39|-3900|-39000000|-39000000000|1985-10-17 00:00:00|1.1111|89|089|X'5959595959'|X'59' +90|1|-40|-4000|-40000000|-40000000000|1985-10-18 00:00:00|1.0989|90|090|X'5A'|X'5A' +91|0|-41|-4100|-41000000|-41000000000|1985-10-19 00:00:00|1.0870|91|091|X'5B5B'|X'5B' +92|1|-42|-4200|-42000000|-42000000000|1985-10-20 00:00:00|1.0753|92|092|X'5C5C5C'|X'5C' +93|0|-43|-4300|-43000000|-43000000000|1985-10-21 00:00:00|1.0638|93|093|X'5D5D5D5D'|X'5D' +94|1|-44|-4400|-44000000|-44000000000|1985-10-22 00:00:00|1.0526|94|094|X'5E5E5E5E5E'|X'5E' +95|0|-45|-4500|-45000000|-45000000000|1985-10-23 00:00:00|1.0417|95|095|X'5F'|X'5F' +96|1|-46|-4600|-46000000|-46000000000|1985-10-24 00:00:00|1.0309|96|096|X'6060'|X'60' +97|0|-47|-4700|-47000000|-47000000000|1985-10-25 00:00:00|1.0204|97|097|X'616161'|X'61' +98|1|-48|-4800|-48000000|-48000000000|1985-10-26 00:00:00|1.0101|98|098|X'62626262'|X'62' +99|0|-49|-4900|-49000000|-49000000000|1985-10-27 00:00:00|1.0000|99|099|X'6363636363'|X'63'