mirror of
https://github.com/cldellow/sqlite-parquet-vtable.git
synced 2025-09-08 22:24:56 +00:00
Add a pg oracle
- define `datetime`, `printf` fns in pg so it produces similar output as sqlite - tidy up input data to be less wide To do: some fns to make it easy to generate a new test case. Probably want to mount all the 3 parquets simultaneously and refer to the sqlite table by the same name as the pg table.
This commit is contained in:
Colin Dellow
Binary file not shown.
Binary file not shown.
Binary file not shown.
55
parquet-generator/create-pg-db
Executable file
55
parquet-generator/create-pg-db
Executable file
@@ -0,0 +1,55 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
export PGUSER=postgres
|
||||
|
||||
dropdb parquet || true
|
||||
createdb parquet
|
||||
|
||||
here=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")
|
||||
|
||||
psql parquet << EOF
|
||||
CREATE TABLE no_nulls(
|
||||
rowid INT2,
|
||||
bool_0 INT2,
|
||||
int8_1 INT2,
|
||||
int16_2 INT2,
|
||||
int32_3 INT4,
|
||||
int64_4 INT8,
|
||||
ts_5 int8,
|
||||
double_6 DOUBLE PRECISION,
|
||||
string_7 TEXT,
|
||||
string_8 TEXT,
|
||||
binary_9 TEXT,
|
||||
binary_10 TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE nulls (LIKE no_nulls);
|
||||
|
||||
COPY no_nulls FROM '$here/no-nulls.csv';
|
||||
COPY nulls FROM '$here/nulls.csv';
|
||||
|
||||
CREATE OR REPLACE FUNCTION quote(TEXT) RETURNS TEXT AS \$\$
|
||||
BEGIN
|
||||
RETURN \$1;
|
||||
END;
|
||||
\$\$ LANGUAGE 'plpgsql' IMMUTABLE;
|
||||
|
||||
CREATE OR REPLACE FUNCTION datetime(INT8, TEXT) RETURNS TEXT AS \$\$
|
||||
BEGIN
|
||||
RETURN to_timestamp(\$1) AT TIME ZONE 'UTC';
|
||||
END;
|
||||
\$\$ LANGUAGE 'plpgsql' IMMUTABLE;
|
||||
|
||||
CREATE OR REPLACE FUNCTION printf(TEXT, DOUBLE PRECISION) RETURNS TEXT AS \$\$
|
||||
BEGIN
|
||||
RETURN CAST(\$2 AS DECIMAL(10, 4))::TEXT;
|
||||
END;
|
||||
\$\$ LANGUAGE 'plpgsql' IMMUTABLE;
|
||||
|
||||
|
||||
|
||||
--COPY (SELECT rowid, bool_0, int8_1, int16_2, int32_3, int64_4, datetime(ts_5 / 1000, 'unixepoch'), double_6, string_7, string_8, quote(binary_9), quote(binary_10) FROM nulls LIMIT 3) TO STDOUT WITH (DELIMITER '|', NULL '');
|
||||
COPY (SELECT rowid, bool_0, int8_1, int16_2, int32_3, int64_4, datetime(ts_5 / 1000, 'unixepoch'), printf('%.4f', double_6) AS double_6, string_7, string_8, quote(binary_9), quote(binary_10) FROM no_nulls) TO STDOUT WITH (DELIMITER '|', NULL '');
|
||||
EOF
|
||||
100
parquet-generator/no-nulls.csv
Normal file
100
parquet-generator/no-nulls.csv
Normal file
@@ -0,0 +1,100 @@
|
||||
0 1 50 5000 50000000 50000000000 490665600000 100.0 0 000 X'00' X'00'
|
||||
1 0 49 4900 49000000 49000000000 490752000000 50.0 1 001 X'0101' X'01'
|
||||
2 1 48 4800 48000000 48000000000 490838400000 33.333333333333336 2 002 X'020202' X'02'
|
||||
3 0 47 4700 47000000 47000000000 490924800000 25.0 3 003 X'03030303' X'03'
|
||||
4 1 46 4600 46000000 46000000000 491011200000 20.0 4 004 X'0404040404' X'04'
|
||||
5 0 45 4500 45000000 45000000000 491097600000 16.666666666666668 5 005 X'05' X'05'
|
||||
6 1 44 4400 44000000 44000000000 491184000000 14.285714285714286 6 006 X'0606' X'06'
|
||||
7 0 43 4300 43000000 43000000000 491270400000 12.5 7 007 X'070707' X'07'
|
||||
8 1 42 4200 42000000 42000000000 491356800000 11.11111111111111 8 008 X'08080808' X'08'
|
||||
9 0 41 4100 41000000 41000000000 491443200000 10.0 9 009 X'0909090909' X'09'
|
||||
10 1 40 4000 40000000 40000000000 491529600000 9.090909090909092 10 010 X'0A' X'0A'
|
||||
11 0 39 3900 39000000 39000000000 491616000000 8.333333333333334 11 011 X'0B0B' X'0B'
|
||||
12 1 38 3800 38000000 38000000000 491702400000 7.6923076923076925 12 012 X'0C0C0C' X'0C'
|
||||
13 0 37 3700 37000000 37000000000 491788800000 7.142857142857143 13 013 X'0D0D0D0D' X'0D'
|
||||
14 1 36 3600 36000000 36000000000 491875200000 6.666666666666667 14 014 X'0E0E0E0E0E' X'0E'
|
||||
15 0 35 3500 35000000 35000000000 491961600000 6.25 15 015 X'0F' X'0F'
|
||||
16 1 34 3400 34000000 34000000000 492048000000 5.882352941176471 16 016 X'1010' X'10'
|
||||
17 0 33 3300 33000000 33000000000 492134400000 5.555555555555555 17 017 X'111111' X'11'
|
||||
18 1 32 3200 32000000 32000000000 492220800000 5.2631578947368425 18 018 X'12121212' X'12'
|
||||
19 0 31 3100 31000000 31000000000 492307200000 5.0 19 019 X'1313131313' X'13'
|
||||
20 1 30 3000 30000000 30000000000 492393600000 4.761904761904762 20 020 X'14' X'14'
|
||||
21 0 29 2900 29000000 29000000000 492480000000 4.545454545454546 21 021 X'1515' X'15'
|
||||
22 1 28 2800 28000000 28000000000 492566400000 4.3478260869565215 22 022 X'161616' X'16'
|
||||
23 0 27 2700 27000000 27000000000 492652800000 4.166666666666667 23 023 X'17171717' X'17'
|
||||
24 1 26 2600 26000000 26000000000 492739200000 4.0 24 024 X'1818181818' X'18'
|
||||
25 0 25 2500 25000000 25000000000 492825600000 3.8461538461538463 25 025 X'19' X'19'
|
||||
26 1 24 2400 24000000 24000000000 492912000000 3.7037037037037037 26 026 X'1A1A' X'1A'
|
||||
27 0 23 2300 23000000 23000000000 492998400000 3.5714285714285716 27 027 X'1B1B1B' X'1B'
|
||||
28 1 22 2200 22000000 22000000000 493084800000 3.4482758620689653 28 028 X'1C1C1C1C' X'1C'
|
||||
29 0 21 2100 21000000 21000000000 493171200000 3.3333333333333335 29 029 X'1D1D1D1D1D' X'1D'
|
||||
30 1 20 2000 20000000 20000000000 493257600000 3.225806451612903 30 030 X'1E' X'1E'
|
||||
31 0 19 1900 19000000 19000000000 493344000000 3.125 31 031 X'1F1F' X'1F'
|
||||
32 1 18 1800 18000000 18000000000 493430400000 3.0303030303030303 32 032 X'202020' X'20'
|
||||
33 0 17 1700 17000000 17000000000 493516800000 2.9411764705882355 33 033 X'21212121' X'21'
|
||||
34 1 16 1600 16000000 16000000000 493603200000 2.857142857142857 34 034 X'2222222222' X'22'
|
||||
35 0 15 1500 15000000 15000000000 493689600000 2.7777777777777777 35 035 X'23' X'23'
|
||||
36 1 14 1400 14000000 14000000000 493776000000 2.7027027027027026 36 036 X'2424' X'24'
|
||||
37 0 13 1300 13000000 13000000000 493862400000 2.6315789473684212 37 037 X'252525' X'25'
|
||||
38 1 12 1200 12000000 12000000000 493948800000 2.5641025641025643 38 038 X'26262626' X'26'
|
||||
39 0 11 1100 11000000 11000000000 494035200000 2.5 39 039 X'2727272727' X'27'
|
||||
40 1 10 1000 10000000 10000000000 494121600000 2.4390243902439024 40 040 X'28' X'28'
|
||||
41 0 9 900 9000000 9000000000 494208000000 2.380952380952381 41 041 X'2929' X'29'
|
||||
42 1 8 800 8000000 8000000000 494294400000 2.3255813953488373 42 042 X'2A2A2A' X'2A'
|
||||
43 0 7 700 7000000 7000000000 494380800000 2.272727272727273 43 043 X'2B2B2B2B' X'2B'
|
||||
44 1 6 600 6000000 6000000000 494467200000 2.2222222222222223 44 044 X'2C2C2C2C2C' X'2C'
|
||||
45 0 5 500 5000000 5000000000 494553600000 2.1739130434782608 45 045 X'2D' X'2D'
|
||||
46 1 4 400 4000000 4000000000 494640000000 2.127659574468085 46 046 X'2E2E' X'2E'
|
||||
47 0 3 300 3000000 3000000000 494726400000 2.0833333333333335 47 047 X'2F2F2F' X'2F'
|
||||
48 1 2 200 2000000 2000000000 494812800000 2.0408163265306123 48 048 X'30303030' X'30'
|
||||
49 0 1 100 1000000 1000000000 494899200000 2.0 49 049 X'3131313131' X'31'
|
||||
50 1 0 0 0 0 494985600000 1.9607843137254901 50 050 X'32' X'32'
|
||||
51 0 -1 -100 -1000000 -1000000000 495072000000 1.9230769230769231 51 051 X'3333' X'33'
|
||||
52 1 -2 -200 -2000000 -2000000000 495158400000 1.8867924528301887 52 052 X'343434' X'34'
|
||||
53 0 -3 -300 -3000000 -3000000000 495244800000 1.8518518518518519 53 053 X'35353535' X'35'
|
||||
54 1 -4 -400 -4000000 -4000000000 495331200000 1.8181818181818181 54 054 X'3636363636' X'36'
|
||||
55 0 -5 -500 -5000000 -5000000000 495417600000 1.7857142857142858 55 055 X'37' X'37'
|
||||
56 1 -6 -600 -6000000 -6000000000 495504000000 1.7543859649122806 56 056 X'3838' X'38'
|
||||
57 0 -7 -700 -7000000 -7000000000 495590400000 1.7241379310344827 57 057 X'393939' X'39'
|
||||
58 1 -8 -800 -8000000 -8000000000 495676800000 1.694915254237288 58 058 X'3A3A3A3A' X'3A'
|
||||
59 0 -9 -900 -9000000 -9000000000 495763200000 1.6666666666666667 59 059 X'3B3B3B3B3B' X'3B'
|
||||
60 1 -10 -1000 -10000000 -10000000000 495849600000 1.639344262295082 60 060 X'3C' X'3C'
|
||||
61 0 -11 -1100 -11000000 -11000000000 495936000000 1.6129032258064515 61 061 X'3D3D' X'3D'
|
||||
62 1 -12 -1200 -12000000 -12000000000 496022400000 1.5873015873015872 62 062 X'3E3E3E' X'3E'
|
||||
63 0 -13 -1300 -13000000 -13000000000 496108800000 1.5625 63 063 X'3F3F3F3F' X'3F'
|
||||
64 1 -14 -1400 -14000000 -14000000000 496195200000 1.5384615384615385 64 064 X'4040404040' X'40'
|
||||
65 0 -15 -1500 -15000000 -15000000000 496281600000 1.5151515151515151 65 065 X'41' X'41'
|
||||
66 1 -16 -1600 -16000000 -16000000000 496368000000 1.492537313432836 66 066 X'4242' X'42'
|
||||
67 0 -17 -1700 -17000000 -17000000000 496454400000 1.4705882352941178 67 067 X'434343' X'43'
|
||||
68 1 -18 -1800 -18000000 -18000000000 496540800000 1.4492753623188406 68 068 X'44444444' X'44'
|
||||
69 0 -19 -1900 -19000000 -19000000000 496627200000 1.4285714285714286 69 069 X'4545454545' X'45'
|
||||
70 1 -20 -2000 -20000000 -20000000000 496713600000 1.408450704225352 70 070 X'46' X'46'
|
||||
71 0 -21 -2100 -21000000 -21000000000 496800000000 1.3888888888888888 71 071 X'4747' X'47'
|
||||
72 1 -22 -2200 -22000000 -22000000000 496886400000 1.36986301369863 72 072 X'484848' X'48'
|
||||
73 0 -23 -2300 -23000000 -23000000000 496972800000 1.3513513513513513 73 073 X'49494949' X'49'
|
||||
74 1 -24 -2400 -24000000 -24000000000 497059200000 1.3333333333333333 74 074 X'4A4A4A4A4A' X'4A'
|
||||
75 0 -25 -2500 -25000000 -25000000000 497145600000 1.3157894736842106 75 075 X'4B' X'4B'
|
||||
76 1 -26 -2600 -26000000 -26000000000 497232000000 1.2987012987012987 76 076 X'4C4C' X'4C'
|
||||
77 0 -27 -2700 -27000000 -27000000000 497318400000 1.2820512820512822 77 077 X'4D4D4D' X'4D'
|
||||
78 1 -28 -2800 -28000000 -28000000000 497404800000 1.2658227848101267 78 078 X'4E4E4E4E' X'4E'
|
||||
79 0 -29 -2900 -29000000 -29000000000 497491200000 1.25 79 079 X'4F4F4F4F4F' X'4F'
|
||||
80 1 -30 -3000 -30000000 -30000000000 497577600000 1.2345679012345678 80 080 X'50' X'50'
|
||||
81 0 -31 -3100 -31000000 -31000000000 497664000000 1.2195121951219512 81 081 X'5151' X'51'
|
||||
82 1 -32 -3200 -32000000 -32000000000 497750400000 1.2048192771084338 82 082 X'525252' X'52'
|
||||
83 0 -33 -3300 -33000000 -33000000000 497836800000 1.1904761904761905 83 083 X'53535353' X'53'
|
||||
84 1 -34 -3400 -34000000 -34000000000 497923200000 1.1764705882352942 84 084 X'5454545454' X'54'
|
||||
85 0 -35 -3500 -35000000 -35000000000 498009600000 1.1627906976744187 85 085 X'55' X'55'
|
||||
86 1 -36 -3600 -36000000 -36000000000 498096000000 1.1494252873563218 86 086 X'5656' X'56'
|
||||
87 0 -37 -3700 -37000000 -37000000000 498182400000 1.1363636363636365 87 087 X'575757' X'57'
|
||||
88 1 -38 -3800 -38000000 -38000000000 498268800000 1.1235955056179776 88 088 X'58585858' X'58'
|
||||
89 0 -39 -3900 -39000000 -39000000000 498355200000 1.1111111111111112 89 089 X'5959595959' X'59'
|
||||
90 1 -40 -4000 -40000000 -40000000000 498441600000 1.098901098901099 90 090 X'5A' X'5A'
|
||||
91 0 -41 -4100 -41000000 -41000000000 498528000000 1.0869565217391304 91 091 X'5B5B' X'5B'
|
||||
92 1 -42 -4200 -42000000 -42000000000 498614400000 1.075268817204301 92 092 X'5C5C5C' X'5C'
|
||||
93 0 -43 -4300 -43000000 -43000000000 498700800000 1.0638297872340425 93 093 X'5D5D5D5D' X'5D'
|
||||
94 1 -44 -4400 -44000000 -44000000000 498787200000 1.0526315789473684 94 094 X'5E5E5E5E5E' X'5E'
|
||||
95 0 -45 -4500 -45000000 -45000000000 498873600000 1.0416666666666667 95 095 X'5F' X'5F'
|
||||
96 1 -46 -4600 -46000000 -46000000000 498960000000 1.0309278350515463 96 096 X'6060' X'60'
|
||||
97 0 -47 -4700 -47000000 -47000000000 499046400000 1.0204081632653061 97 097 X'616161' X'61'
|
||||
98 1 -48 -4800 -48000000 -48000000000 499132800000 1.0101010101010102 98 098 X'62626262' X'62'
|
||||
99 0 -49 -4900 -49000000 -49000000000 499219200000 1 99 099 X'6363636363' X'63'
|
||||
|
100
parquet-generator/nulls.csv
Normal file
100
parquet-generator/nulls.csv
Normal file
@@ -0,0 +1,100 @@
|
||||
0 1 50 5000 50000000 50000000000 490665600000 100.0 0 000 X'00' X'00'
|
||||
1 0 49 4900 49000000 49000000000 490752000000 50.0 1 001 X'0101' X'01'
|
||||
2 1 48 4800 48000000 48000000000 490838400000 33.333333333333336 2 002 X'020202' X'02'
|
||||
3 0 47 4700 47000000 47000000000 490924800000 25.0 3 003 X'03030303' X'03'
|
||||
4 1 46 4600 46000000 46000000000 491011200000 20.0 4 004 X'0404040404' X'04'
|
||||
5 0 45 4500 45000000 45000000000 491097600000 16.666666666666668 5 005 X'05' X'05'
|
||||
6 1 44 4400 44000000 44000000000 491184000000 14.285714285714286 6 006 X'0606' X'06'
|
||||
7 0 43 4300 43000000 43000000000 491270400000 12.5 7 007 X'070707' X'07'
|
||||
8 1 42 4200 42000000 42000000000 491356800000 11.11111111111111 8 008 X'08080808' X'08'
|
||||
9 0 41 4100 41000000 41000000000 491443200000 10.0 9 009 X'0909090909' X'09'
|
||||
10 \N \N \N \N \N \N \N \N \N \N \N
|
||||
11 \N \N \N \N \N \N \N \N \N \N \N
|
||||
12 \N \N \N \N \N \N \N \N \N \N \N
|
||||
13 \N \N \N \N \N \N \N \N \N \N \N
|
||||
14 \N \N \N \N \N \N \N \N \N \N \N
|
||||
15 \N \N \N \N \N \N \N \N \N \N \N
|
||||
16 \N \N \N \N \N \N \N \N \N \N \N
|
||||
17 \N \N \N \N \N \N \N \N \N \N \N
|
||||
18 \N \N \N \N \N \N \N \N \N \N \N
|
||||
19 \N \N \N \N \N \N \N \N \N \N \N
|
||||
20 \N 30 \N 30000000 \N 492393600000 \N 20 \N X'14' \N
|
||||
21 0 \N 2900 \N 29000000000 \N 4.545454545454546 \N 021 \N X'15'
|
||||
22 \N 28 \N 28000000 \N 492566400000 \N 22 \N X'161616' \N
|
||||
23 0 \N 2700 \N 27000000000 \N 4.166666666666667 \N 023 \N X'17'
|
||||
24 \N 26 \N 26000000 \N 492739200000 \N 24 \N X'1818181818' \N
|
||||
25 0 \N 2500 \N 25000000000 \N 3.8461538461538463 \N 025 \N X'19'
|
||||
26 \N 24 \N 24000000 \N 492912000000 \N 26 \N X'1A1A' \N
|
||||
27 0 \N 2300 \N 23000000000 \N 3.5714285714285716 \N 027 \N X'1B'
|
||||
28 \N 22 \N 22000000 \N 493084800000 \N 28 \N X'1C1C1C1C' \N
|
||||
29 0 \N 2100 \N 21000000000 \N 3.3333333333333335 \N 029 \N X'1D'
|
||||
30 \N 20 \N 20000000 \N 493257600000 \N 30 \N X'1E' \N
|
||||
31 0 \N 1900 \N 19000000000 \N 3.125 \N 031 \N X'1F'
|
||||
32 \N 18 \N 18000000 \N 493430400000 \N 32 \N X'202020' \N
|
||||
33 0 \N 1700 \N 17000000000 \N 2.9411764705882355 \N 033 \N X'21'
|
||||
34 \N 16 \N 16000000 \N 493603200000 \N 34 \N X'2222222222' \N
|
||||
35 0 \N 1500 \N 15000000000 \N 2.7777777777777777 \N 035 \N X'23'
|
||||
36 \N 14 \N 14000000 \N 493776000000 \N 36 \N X'2424' \N
|
||||
37 0 \N 1300 \N 13000000000 \N 2.6315789473684212 \N 037 \N X'25'
|
||||
38 \N 12 \N 12000000 \N 493948800000 \N 38 \N X'26262626' \N
|
||||
39 0 \N 1100 \N 11000000000 \N 2.5 \N 039 \N X'27'
|
||||
40 \N 10 \N 10000000 \N 494121600000 \N 40 \N X'28' \N
|
||||
41 0 \N 900 \N 9000000000 \N 2.380952380952381 \N 041 \N X'29'
|
||||
42 \N 8 \N 8000000 \N 494294400000 \N 42 \N X'2A2A2A' \N
|
||||
43 0 \N 700 \N 7000000000 \N 2.272727272727273 \N 043 \N X'2B'
|
||||
44 \N 6 \N 6000000 \N 494467200000 \N 44 \N X'2C2C2C2C2C' \N
|
||||
45 0 \N 500 \N 5000000000 \N 2.1739130434782608 \N 045 \N X'2D'
|
||||
46 \N 4 \N 4000000 \N 494640000000 \N 46 \N X'2E2E' \N
|
||||
47 0 \N 300 \N 3000000000 \N 2.0833333333333335 \N 047 \N X'2F'
|
||||
48 \N 2 \N 2000000 \N 494812800000 \N 48 \N X'30303030' \N
|
||||
49 0 \N 100 \N 1000000000 \N 2.0 \N 049 \N X'31'
|
||||
50 \N 0 \N 0 \N 494985600000 \N 50 \N X'32' \N
|
||||
51 0 \N -100 \N -1000000000 \N 1.9230769230769231 \N 051 \N X'33'
|
||||
52 \N -2 \N -2000000 \N 495158400000 \N 52 \N X'343434' \N
|
||||
53 0 \N -300 \N -3000000000 \N 1.8518518518518519 \N 053 \N X'35'
|
||||
54 \N -4 \N -4000000 \N 495331200000 \N 54 \N X'3636363636' \N
|
||||
55 0 \N -500 \N -5000000000 \N 1.7857142857142858 \N 055 \N X'37'
|
||||
56 \N -6 \N -6000000 \N 495504000000 \N 56 \N X'3838' \N
|
||||
57 0 \N -700 \N -7000000000 \N 1.7241379310344827 \N 057 \N X'39'
|
||||
58 \N -8 \N -8000000 \N 495676800000 \N 58 \N X'3A3A3A3A' \N
|
||||
59 0 \N -900 \N -9000000000 \N 1.6666666666666667 \N 059 \N X'3B'
|
||||
60 \N -10 \N -10000000 \N 495849600000 \N 60 \N X'3C' \N
|
||||
61 0 \N -1100 \N -11000000000 \N 1.6129032258064515 \N 061 \N X'3D'
|
||||
62 \N -12 \N -12000000 \N 496022400000 \N 62 \N X'3E3E3E' \N
|
||||
63 0 \N -1300 \N -13000000000 \N 1.5625 \N 063 \N X'3F'
|
||||
64 \N -14 \N -14000000 \N 496195200000 \N 64 \N X'4040404040' \N
|
||||
65 0 \N -1500 \N -15000000000 \N 1.5151515151515151 \N 065 \N X'41'
|
||||
66 \N -16 \N -16000000 \N 496368000000 \N 66 \N X'4242' \N
|
||||
67 0 \N -1700 \N -17000000000 \N 1.4705882352941178 \N 067 \N X'43'
|
||||
68 \N -18 \N -18000000 \N 496540800000 \N 68 \N X'44444444' \N
|
||||
69 0 \N -1900 \N -19000000000 \N 1.4285714285714286 \N 069 \N X'45'
|
||||
70 \N -20 \N -20000000 \N 496713600000 \N 70 \N X'46' \N
|
||||
71 0 \N -2100 \N -21000000000 \N 1.3888888888888888 \N 071 \N X'47'
|
||||
72 \N -22 \N -22000000 \N 496886400000 \N 72 \N X'484848' \N
|
||||
73 0 \N -2300 \N -23000000000 \N 1.3513513513513513 \N 073 \N X'49'
|
||||
74 \N -24 \N -24000000 \N 497059200000 \N 74 \N X'4A4A4A4A4A' \N
|
||||
75 0 \N -2500 \N -25000000000 \N 1.3157894736842106 \N 075 \N X'4B'
|
||||
76 \N -26 \N -26000000 \N 497232000000 \N 76 \N X'4C4C' \N
|
||||
77 0 \N -2700 \N -27000000000 \N 1.2820512820512822 \N 077 \N X'4D'
|
||||
78 \N -28 \N -28000000 \N 497404800000 \N 78 \N X'4E4E4E4E' \N
|
||||
79 0 \N -2900 \N -29000000000 \N 1.25 \N 079 \N X'4F'
|
||||
80 \N -30 \N -30000000 \N 497577600000 \N 80 \N X'50' \N
|
||||
81 0 \N -3100 \N -31000000000 \N 1.2195121951219512 \N 081 \N X'51'
|
||||
82 \N -32 \N -32000000 \N 497750400000 \N 82 \N X'525252' \N
|
||||
83 0 \N -3300 \N -33000000000 \N 1.1904761904761905 \N 083 \N X'53'
|
||||
84 \N -34 \N -34000000 \N 497923200000 \N 84 \N X'5454545454' \N
|
||||
85 0 \N -3500 \N -35000000000 \N 1.1627906976744187 \N 085 \N X'55'
|
||||
86 \N -36 \N -36000000 \N 498096000000 \N 86 \N X'5656' \N
|
||||
87 0 \N -3700 \N -37000000000 \N 1.1363636363636365 \N 087 \N X'57'
|
||||
88 \N -38 \N -38000000 \N 498268800000 \N 88 \N X'58585858' \N
|
||||
89 0 \N -3900 \N -39000000000 \N 1.1111111111111112 \N 089 \N X'59'
|
||||
90 \N -40 \N -40000000 \N 498441600000 \N 90 \N X'5A' \N
|
||||
91 0 \N -4100 \N -41000000000 \N 1.0869565217391304 \N 091 \N X'5B'
|
||||
92 \N -42 \N -42000000 \N 498614400000 \N 92 \N X'5C5C5C' \N
|
||||
93 0 \N -4300 \N -43000000000 \N 1.0638297872340425 \N 093 \N X'5D'
|
||||
94 \N -44 \N -44000000 \N 498787200000 \N 94 \N X'5E5E5E5E5E' \N
|
||||
95 0 \N -4500 \N -45000000000 \N 1.0416666666666667 \N 095 \N X'5F'
|
||||
96 \N -46 \N -46000000 \N 498960000000 \N 96 \N X'6060' \N
|
||||
97 0 \N -4700 \N -47000000000 \N 1.0204081632653061 \N 097 \N X'61'
|
||||
98 \N -48 \N -48000000 \N 499132800000 \N 98 \N X'62626262' \N
|
||||
99 0 \N -4900 \N -49000000000 \N 1 \N 099 \N X'63'
|
||||
|
@@ -1,4 +1,4 @@
|
||||
from datetime import datetime, timedelta
|
||||
from datetime import datetime, timedelta, timezone
|
||||
import re
|
||||
|
||||
import pyarrow as pa
|
||||
@@ -12,8 +12,8 @@ def make_100_rows():
|
||||
ba_fixed = bytearray()
|
||||
ba_fixed.append(i)
|
||||
ba_variable = bytearray()
|
||||
for j in range(i):
|
||||
ba_variable.append(j)
|
||||
for j in range(1 + i % 5):
|
||||
ba_variable.append(i)
|
||||
row = []
|
||||
# BOOLEAN, INT32, INT64, INT96, DOUBLE, BYTE_ARRAY, FLOAT
|
||||
row.append(i % 2 == 0) # BOOLEAN
|
||||
@@ -21,7 +21,7 @@ def make_100_rows():
|
||||
row.append(100 * (50 - i)) # INT32/INT16
|
||||
row.append(1000 * 1000 * (50 - i)) # INT32/INT32
|
||||
row.append(1000 * 1000 * 1000 * (50 - i)) # INT64/INT64
|
||||
row.append(datetime(1985, 7, 20) + timedelta(days=i)) # INT96
|
||||
row.append(datetime(1985, 7, 20, tzinfo=timezone.utc) + timedelta(days=i)) # INT96
|
||||
row.append(100.0 / (i + 1)) # DOUBLE
|
||||
row.append(str(i)) # BYTE_ARRAY/UTF8
|
||||
row.append('{:03}'.format(i)), # BYTE_ARRAY/UTF8
|
||||
@@ -95,12 +95,46 @@ def write_unsupported_parquets():
|
||||
|
||||
write_parquet(file_name, [], [type], row_group_size=1)
|
||||
|
||||
def write_csv(file_name, rows):
|
||||
r'''Write a TSV that can be imported to Postgres.
|
||||
|
||||
Use "\N" for NULLs, tab literal for field separator.'''
|
||||
print('Writing {}'.format(file_name))
|
||||
with open(file_name, 'w') as f:
|
||||
for rowid, row in enumerate(rows):
|
||||
line = str(rowid)
|
||||
for col in row:
|
||||
line += '\t'
|
||||
|
||||
if col == True:
|
||||
line += '1'
|
||||
elif col == False:
|
||||
line += '0'
|
||||
elif col is None:
|
||||
line += r'\N'
|
||||
elif isinstance(col, bytes):
|
||||
# Here we cheat and serialize a string that matches the output of
|
||||
# quote(binary_field) in SQLite
|
||||
entry = r"X'"
|
||||
for b in col:
|
||||
entry += '%0.2X' % b
|
||||
|
||||
entry += "'"
|
||||
line += entry
|
||||
elif isinstance(col, datetime):
|
||||
line += str(1000 * int(col.timestamp()))
|
||||
else:
|
||||
line += str(col)
|
||||
|
||||
f.write(line + '\n')
|
||||
|
||||
def main():
|
||||
'''Entrypoint.'''
|
||||
rows = make_100_rows()
|
||||
types = get_100_rows_types()
|
||||
|
||||
write_parquet('100-rows-1.parquet', rows, types, row_group_size=100)
|
||||
write_csv('no-nulls.csv', rows)
|
||||
write_parquet('100-rows-10.parquet', rows, types, row_group_size=10)
|
||||
|
||||
for i in range(len(rows)):
|
||||
@@ -108,6 +142,7 @@ def main():
|
||||
if (i >= 10 and i <= 19) or (i >= 20 and (i + j) % 2 == 0):
|
||||
rows[i][j] = None
|
||||
write_parquet('100-rows-nulls.parquet', rows, types,row_group_size=10)
|
||||
write_csv('nulls.csv', rows)
|
||||
|
||||
write_unsupported_parquets()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user