setop
/
sqlite-parquet-vtable
mirror of https://github.com/cldellow/sqlite-parquet-vtable.git
1
0
Fork 0
Code Releases Activity

Add query test framework, fix xFilter

This commit is contained in:
Colin Dellow 2018-03-04 21:05:26 -05:00
parent 4c54ab89ae
commit bb3a9440f7
8 changed files with 284 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
parquet/parquet.cc
View File

@ -121,6 +121,7 @@ static int parquetCreate(
*/ */
static int parquetClose(sqlite3_vtab_cursor *cur){ static int parquetClose(sqlite3_vtab_cursor *cur){
sqlite3_vtab_cursor_parquet* p = (sqlite3_vtab_cursor_parquet*)cur; sqlite3_vtab_cursor_parquet* p = (sqlite3_vtab_cursor_parquet*)cur;
p->cursor->close();
delete p->cursor; delete p->cursor;
sqlite3_free(cur); sqlite3_free(cur);
return SQLITE_OK; return SQLITE_OK;
@ -249,28 +250,14 @@ static int parquetEof(sqlite3_vtab_cursor *cur){
** the beginning. ** the beginning.
*/ */
static int parquetFilter( static int parquetFilter(
sqlite3_vtab_cursor *pVtabCursor, sqlite3_vtab_cursor *cur,
int idxNum, const char *idxStr, int idxNum, const char *idxStr,
int argc, sqlite3_value **argv int argc, sqlite3_value **argv
){ ){
printf("xFilter\n"); printf("xFilter\n");
//sqlite3_vtab_cursor_parquet *pCur = (sqlite3_vtab_cursor_parquet*)pVtabCursor; ParquetCursor* cursor = ((sqlite3_vtab_cursor_parquet*)cur)->cursor;
//sqlite3_vtab_parquet *pTab = (sqlite3_vtab_parquet*)pVtabCursor->pVtab; cursor->reset();
return parquetNext(cur);
/*
pCur->iRowid = 0;
if( pCur->rdr.in==0 ){
assert( pCur->rdr.zIn==pTab->zData );
assert( pTab->iStart>=0 );
assert( (size_t)pTab->iStart<=pCur->rdr.nIn );
pCur->rdr.iIn = pTab->iStart;
}else{
fseek(pCur->rdr.in, pTab->iStart, SEEK_SET);
pCur->rdr.iIn = 0;
pCur->rdr.nIn = 0;
}
*/
return parquetNext(pVtabCursor);
} }
/* /*

40
parquet/parquet_cursor.cc
View File

@ -2,27 +2,17 @@
ParquetCursor::ParquetCursor(ParquetTable* table) { ParquetCursor::ParquetCursor(ParquetTable* table) {
this->table = table; this->table = table;
this->rowId = -1; reader = NULL;
// TODO: consider having a long lived handle in ParquetTable that can be borrowed reset();
// without incurring the cost of opening the file from scratch twice
this->reader = parquet::ParquetFileReader::OpenFile(this->table->file.data());
this->rowGroupId = -1;
// TODO: handle the case where rowgroups have disjoint schemas?
// TODO: or at least, fail fast if detected
this->rowsLeftInRowGroup = 0;
this->numRows = reader->metadata()->num_rows();
this->numRowGroups = reader->metadata()->num_row_groups();
} }
bool ParquetCursor::nextRowGroup() { bool ParquetCursor::nextRowGroup() {
// TODO: skip row groups that cannot satisfy the constraints // TODO: skip row groups that cannot satisfy the constraints
if((this->rowGroupId + 1) >= this->numRowGroups) if((rowGroupId + 1) >= numRowGroups)
return false; return false;
rowGroupId++; rowGroupId++;
rowGroupMetadata = this->reader->metadata()->RowGroup(0); rowGroupMetadata = reader->metadata()->RowGroup(0);
rowsLeftInRowGroup = rowGroupMetadata->num_rows(); rowsLeftInRowGroup = rowGroupMetadata->num_rows();
rowGroup = reader->RowGroup(rowGroupId); rowGroup = reader->RowGroup(rowGroupId);
for(unsigned int i = 0; i < scanners.size(); i++) for(unsigned int i = 0; i < scanners.size(); i++)
@ -235,3 +225,25 @@ parquet::Type::type ParquetCursor::getPhysicalType(int col) {
parquet::LogicalType::type ParquetCursor::getLogicalType(int col) { parquet::LogicalType::type ParquetCursor::getLogicalType(int col) {
return logicalTypes[col]; return logicalTypes[col];
} }
void ParquetCursor::close() {
if(reader != NULL) {
reader->Close();
}
}
void ParquetCursor::reset() {
close();
rowId = -1;
// TODO: consider having a long lived handle in ParquetTable that can be borrowed
// without incurring the cost of opening the file from scratch twice
reader = parquet::ParquetFileReader::OpenFile(table->file.data());
rowGroupId = -1;
// TODO: handle the case where rowgroups have disjoint schemas?
// TODO: or at least, fail fast if detected
rowsLeftInRowGroup = 0;
numRows = reader->metadata()->num_rows();
numRowGroups = reader->metadata()->num_row_groups();
}

2
parquet/parquet_cursor.h
View File

@ -32,6 +32,8 @@ public:
ParquetCursor(ParquetTable* table); ParquetCursor(ParquetTable* table);
int getRowId(); int getRowId();
void next(); void next();
void close();
void reset();
bool eof(); bool eof();
void ensureColumn(int col); void ensureColumn(int col);

103
tests/queries/001-table-scan-1-rowgroup.sql Normal file
View File

@ -0,0 +1,103 @@
100-rows-1.parquet
select * from test;
1|50|5000|50000000|50000000000|490665600000|100.0|0|000||
0|49|4900|49000000|49000000000|490752000000|50.0|1|001||
1|48|4800|48000000|48000000000|490838400000|33.3333333333333|2|002||
0|47|4700|47000000|47000000000|490924800000|25.0|3|003||
1|46|4600|46000000|46000000000|491011200000|20.0|4|004||
0|45|4500|45000000|45000000000|491097600000|16.6666666666667|5|005||
1|44|4400|44000000|44000000000|491184000000|14.2857142857143|6|006||
0|43|4300|43000000|43000000000|491270400000|12.5|7|007||
1|42|4200|42000000|42000000000|491356800000|11.1111111111111|8|008||
0|41|4100|41000000|41000000000|491443200000|10.0|9|009||
1|40|4000|40000000|40000000000|491529600000|9.09090909090909|10|010||
0|39|3900|39000000|39000000000|491616000000|8.33333333333333|11|011||
1|38|3800|38000000|38000000000|491702400000|7.69230769230769|12|012||
0|37|3700|37000000|37000000000|491788800000|7.14285714285714|13|013||
1|36|3600|36000000|36000000000|491875200000|6.66666666666667|14|014||
0|35|3500|35000000|35000000000|491961600000|6.25|15|015||
1|34|3400|34000000|34000000000|492048000000|5.88235294117647|16|016||
0|33|3300|33000000|33000000000|492134400000|5.55555555555556|17|017||
1|32|3200|32000000|32000000000|492220800000|5.26315789473684|18|018||
0|31|3100|31000000|31000000000|492307200000|5.0|19|019||
1|30|3000|30000000|30000000000|492393600000|4.76190476190476|20|020||
0|29|2900|29000000|29000000000|492480000000|4.54545454545455|21|021||
1|28|2800|28000000|28000000000|492566400000|4.34782608695652|22|022||
0|27|2700|27000000|27000000000|492652800000|4.16666666666667|23|023||
1|26|2600|26000000|26000000000|492739200000|4.0|24|024||
0|25|2500|25000000|25000000000|492825600000|3.84615384615385|25|025||
1|24|2400|24000000|24000000000|492912000000|3.7037037037037|26|026||
0|23|2300|23000000|23000000000|492998400000|3.57142857142857|27|027||
1|22|2200|22000000|22000000000|493084800000|3.44827586206897|28|028||
0|21|2100|21000000|21000000000|493171200000|3.33333333333333|29|029||
1|20|2000|20000000|20000000000|493257600000|3.2258064516129|30|030||
0|19|1900|19000000|19000000000|493344000000|3.125|31|031||
1|18|1800|18000000|18000000000|493430400000|3.03030303030303|32|032||
0|17|1700|17000000|17000000000|493516800000|2.94117647058824|33|033||!
1|16|1600|16000000|16000000000|493603200000|2.85714285714286|34|034||"
0|15|1500|15000000|15000000000|493689600000|2.77777777777778|35|035||#
1|14|1400|14000000|14000000000|493776000000|2.7027027027027|36|036||$
0|13|1300|13000000|13000000000|493862400000|2.63157894736842|37|037||%
1|12|1200|12000000|12000000000|493948800000|2.56410256410256|38|038||&
0|11|1100|11000000|11000000000|494035200000|2.5|39|039||'
1|10|1000|10000000|10000000000|494121600000|2.4390243902439|40|040||(
0|9|900|9000000|9000000000|494208000000|2.38095238095238|41|041||)
1|8|800|8000000|8000000000|494294400000|2.32558139534884|42|042||*
0|7|700|7000000|7000000000|494380800000|2.27272727272727|43|043||+
1|6|600|6000000|6000000000|494467200000|2.22222222222222|44|044||,
0|5|500|5000000|5000000000|494553600000|2.17391304347826|45|045||-
1|4|400|4000000|4000000000|494640000000|2.12765957446809|46|046||.
0|3|300|3000000|3000000000|494726400000|2.08333333333333|47|047||/
1|2|200|2000000|2000000000|494812800000|2.04081632653061|48|048||0
0|1|100|1000000|1000000000|494899200000|2.0|49|049||1
1|0|0|0|0|494985600000|1.96078431372549|50|050||2
0|-1|-100|-1000000|-1000000000|495072000000|1.92307692307692|51|051||3
1|-2|-200|-2000000|-2000000000|495158400000|1.88679245283019|52|052||4
0|-3|-300|-3000000|-3000000000|495244800000|1.85185185185185|53|053||5
1|-4|-400|-4000000|-4000000000|495331200000|1.81818181818182|54|054||6
0|-5|-500|-5000000|-5000000000|495417600000|1.78571428571429|55|055||7
1|-6|-600|-6000000|-6000000000|495504000000|1.75438596491228|56|056||8
0|-7|-700|-7000000|-7000000000|495590400000|1.72413793103448|57|057||9
1|-8|-800|-8000000|-8000000000|495676800000|1.69491525423729|58|058||:
0|-9|-900|-9000000|-9000000000|495763200000|1.66666666666667|59|059||;
1|-10|-1000|-10000000|-10000000000|495849600000|1.63934426229508|60|060||<
0|-11|-1100|-11000000|-11000000000|495936000000|1.61290322580645|61|061||=
1|-12|-1200|-12000000|-12000000000|496022400000|1.58730158730159|62|062||>
0|-13|-1300|-13000000|-13000000000|496108800000|1.5625|63|063||?
1|-14|-1400|-14000000|-14000000000|496195200000|1.53846153846154|64|064||@
0|-15|-1500|-15000000|-15000000000|496281600000|1.51515151515152|65|065||A
1|-16|-1600|-16000000|-16000000000|496368000000|1.49253731343284|66|066||B
0|-17|-1700|-17000000|-17000000000|496454400000|1.47058823529412|67|067||C
1|-18|-1800|-18000000|-18000000000|496540800000|1.44927536231884|68|068||D
0|-19|-1900|-19000000|-19000000000|496627200000|1.42857142857143|69|069||E
1|-20|-2000|-20000000|-20000000000|496713600000|1.40845070422535|70|070||F
0|-21|-2100|-21000000|-21000000000|496800000000|1.38888888888889|71|071||G
1|-22|-2200|-22000000|-22000000000|496886400000|1.36986301369863|72|072||H
0|-23|-2300|-23000000|-23000000000|496972800000|1.35135135135135|73|073||I
1|-24|-2400|-24000000|-24000000000|497059200000|1.33333333333333|74|074||J
0|-25|-2500|-25000000|-25000000000|497145600000|1.31578947368421|75|075||K
1|-26|-2600|-26000000|-26000000000|497232000000|1.2987012987013|76|076||L
0|-27|-2700|-27000000|-27000000000|497318400000|1.28205128205128|77|077||M
1|-28|-2800|-28000000|-28000000000|497404800000|1.26582278481013|78|078||N
0|-29|-2900|-29000000|-29000000000|497491200000|1.25|79|079||O
1|-30|-3000|-30000000|-30000000000|497577600000|1.23456790123457|80|080||P
0|-31|-3100|-31000000|-31000000000|497664000000|1.21951219512195|81|081||Q
1|-32|-3200|-32000000|-32000000000|497750400000|1.20481927710843|82|082||R
0|-33|-3300|-33000000|-33000000000|497836800000|1.19047619047619|83|083||S
1|-34|-3400|-34000000|-34000000000|497923200000|1.17647058823529|84|084||T
0|-35|-3500|-35000000|-35000000000|498009600000|1.16279069767442|85|085||U
1|-36|-3600|-36000000|-36000000000|498096000000|1.14942528735632|86|086||V
0|-37|-3700|-37000000|-37000000000|498182400000|1.13636363636364|87|087||W
1|-38|-3800|-38000000|-38000000000|498268800000|1.12359550561798|88|088||X
0|-39|-3900|-39000000|-39000000000|498355200000|1.11111111111111|89|089||Y
1|-40|-4000|-40000000|-40000000000|498441600000|1.0989010989011|90|090||Z
0|-41|-4100|-41000000|-41000000000|498528000000|1.08695652173913|91|091||[
1|-42|-4200|-42000000|-42000000000|498614400000|1.0752688172043|92|092||\
0|-43|-4300|-43000000|-43000000000|498700800000|1.06382978723404|93|093||]
1|-44|-4400|-44000000|-44000000000|498787200000|1.05263157894737|94|094||^
0|-45|-4500|-45000000|-45000000000|498873600000|1.04166666666667|95|095||_
1|-46|-4600|-46000000|-46000000000|498960000000|1.03092783505155|96|096||`
0|-47|-4700|-47000000|-47000000000|499046400000|1.02040816326531|97|097||a
1|-48|-4800|-48000000|-48000000000|499132800000|1.01010101010101|98|098||b
0|-49|-4900|-49000000|-49000000000|499219200000|1.0|99|099||c

103
tests/queries/002-table-scan-10-rowgroups.sql Normal file
View File

@ -0,0 +1,103 @@
100-rows-10.parquet
select * from test;
1|50|5000|50000000|50000000000|490665600000|100.0|0|000||
0|49|4900|49000000|49000000000|490752000000|50.0|1|001||
1|48|4800|48000000|48000000000|490838400000|33.3333333333333|2|002||
0|47|4700|47000000|47000000000|490924800000|25.0|3|003||
1|46|4600|46000000|46000000000|491011200000|20.0|4|004||
0|45|4500|45000000|45000000000|491097600000|16.6666666666667|5|005||
1|44|4400|44000000|44000000000|491184000000|14.2857142857143|6|006||
0|43|4300|43000000|43000000000|491270400000|12.5|7|007||
1|42|4200|42000000|42000000000|491356800000|11.1111111111111|8|008||
0|41|4100|41000000|41000000000|491443200000|10.0|9|009||
1|40|4000|40000000|40000000000|491529600000|9.09090909090909|10|010||
0|39|3900|39000000|39000000000|491616000000|8.33333333333333|11|011||
1|38|3800|38000000|38000000000|491702400000|7.69230769230769|12|012||
0|37|3700|37000000|37000000000|491788800000|7.14285714285714|13|013||
1|36|3600|36000000|36000000000|491875200000|6.66666666666667|14|014||
0|35|3500|35000000|35000000000|491961600000|6.25|15|015||
1|34|3400|34000000|34000000000|492048000000|5.88235294117647|16|016||
0|33|3300|33000000|33000000000|492134400000|5.55555555555556|17|017||
1|32|3200|32000000|32000000000|492220800000|5.26315789473684|18|018||
0|31|3100|31000000|31000000000|492307200000|5.0|19|019||
1|30|3000|30000000|30000000000|492393600000|4.76190476190476|20|020||
0|29|2900|29000000|29000000000|492480000000|4.54545454545455|21|021||
1|28|2800|28000000|28000000000|492566400000|4.34782608695652|22|022||
0|27|2700|27000000|27000000000|492652800000|4.16666666666667|23|023||
1|26|2600|26000000|26000000000|492739200000|4.0|24|024||
0|25|2500|25000000|25000000000|492825600000|3.84615384615385|25|025||
1|24|2400|24000000|24000000000|492912000000|3.7037037037037|26|026||
0|23|2300|23000000|23000000000|492998400000|3.57142857142857|27|027||
1|22|2200|22000000|22000000000|493084800000|3.44827586206897|28|028||
0|21|2100|21000000|21000000000|493171200000|3.33333333333333|29|029||
1|20|2000|20000000|20000000000|493257600000|3.2258064516129|30|030||
0|19|1900|19000000|19000000000|493344000000|3.125|31|031||
1|18|1800|18000000|18000000000|493430400000|3.03030303030303|32|032||
0|17|1700|17000000|17000000000|493516800000|2.94117647058824|33|033||!
1|16|1600|16000000|16000000000|493603200000|2.85714285714286|34|034||"
0|15|1500|15000000|15000000000|493689600000|2.77777777777778|35|035||#
1|14|1400|14000000|14000000000|493776000000|2.7027027027027|36|036||$
0|13|1300|13000000|13000000000|493862400000|2.63157894736842|37|037||%
1|12|1200|12000000|12000000000|493948800000|2.56410256410256|38|038||&
0|11|1100|11000000|11000000000|494035200000|2.5|39|039||'
1|10|1000|10000000|10000000000|494121600000|2.4390243902439|40|040||(
0|9|900|9000000|9000000000|494208000000|2.38095238095238|41|041||)
1|8|800|8000000|8000000000|494294400000|2.32558139534884|42|042||*
0|7|700|7000000|7000000000|494380800000|2.27272727272727|43|043||+
1|6|600|6000000|6000000000|494467200000|2.22222222222222|44|044||,
0|5|500|5000000|5000000000|494553600000|2.17391304347826|45|045||-
1|4|400|4000000|4000000000|494640000000|2.12765957446809|46|046||.
0|3|300|3000000|3000000000|494726400000|2.08333333333333|47|047||/
1|2|200|2000000|2000000000|494812800000|2.04081632653061|48|048||0
0|1|100|1000000|1000000000|494899200000|2.0|49|049||1
1|0|0|0|0|494985600000|1.96078431372549|50|050||2
0|-1|-100|-1000000|-1000000000|495072000000|1.92307692307692|51|051||3
1|-2|-200|-2000000|-2000000000|495158400000|1.88679245283019|52|052||4
0|-3|-300|-3000000|-3000000000|495244800000|1.85185185185185|53|053||5
1|-4|-400|-4000000|-4000000000|495331200000|1.81818181818182|54|054||6
0|-5|-500|-5000000|-5000000000|495417600000|1.78571428571429|55|055||7
1|-6|-600|-6000000|-6000000000|495504000000|1.75438596491228|56|056||8
0|-7|-700|-7000000|-7000000000|495590400000|1.72413793103448|57|057||9
1|-8|-800|-8000000|-8000000000|495676800000|1.69491525423729|58|058||:
0|-9|-900|-9000000|-9000000000|495763200000|1.66666666666667|59|059||;
1|-10|-1000|-10000000|-10000000000|495849600000|1.63934426229508|60|060||<
0|-11|-1100|-11000000|-11000000000|495936000000|1.61290322580645|61|061||=
1|-12|-1200|-12000000|-12000000000|496022400000|1.58730158730159|62|062||>
0|-13|-1300|-13000000|-13000000000|496108800000|1.5625|63|063||?
1|-14|-1400|-14000000|-14000000000|496195200000|1.53846153846154|64|064||@
0|-15|-1500|-15000000|-15000000000|496281600000|1.51515151515152|65|065||A
1|-16|-1600|-16000000|-16000000000|496368000000|1.49253731343284|66|066||B
0|-17|-1700|-17000000|-17000000000|496454400000|1.47058823529412|67|067||C
1|-18|-1800|-18000000|-18000000000|496540800000|1.44927536231884|68|068||D
0|-19|-1900|-19000000|-19000000000|496627200000|1.42857142857143|69|069||E
1|-20|-2000|-20000000|-20000000000|496713600000|1.40845070422535|70|070||F
0|-21|-2100|-21000000|-21000000000|496800000000|1.38888888888889|71|071||G
1|-22|-2200|-22000000|-22000000000|496886400000|1.36986301369863|72|072||H
0|-23|-2300|-23000000|-23000000000|496972800000|1.35135135135135|73|073||I
1|-24|-2400|-24000000|-24000000000|497059200000|1.33333333333333|74|074||J
0|-25|-2500|-25000000|-25000000000|497145600000|1.31578947368421|75|075||K
1|-26|-2600|-26000000|-26000000000|497232000000|1.2987012987013|76|076||L
0|-27|-2700|-27000000|-27000000000|497318400000|1.28205128205128|77|077||M
1|-28|-2800|-28000000|-28000000000|497404800000|1.26582278481013|78|078||N
0|-29|-2900|-29000000|-29000000000|497491200000|1.25|79|079||O
1|-30|-3000|-30000000|-30000000000|497577600000|1.23456790123457|80|080||P
0|-31|-3100|-31000000|-31000000000|497664000000|1.21951219512195|81|081||Q
1|-32|-3200|-32000000|-32000000000|497750400000|1.20481927710843|82|082||R
0|-33|-3300|-33000000|-33000000000|497836800000|1.19047619047619|83|083||S
1|-34|-3400|-34000000|-34000000000|497923200000|1.17647058823529|84|084||T
0|-35|-3500|-35000000|-35000000000|498009600000|1.16279069767442|85|085||U
1|-36|-3600|-36000000|-36000000000|498096000000|1.14942528735632|86|086||V
0|-37|-3700|-37000000|-37000000000|498182400000|1.13636363636364|87|087||W
1|-38|-3800|-38000000|-38000000000|498268800000|1.12359550561798|88|088||X
0|-39|-3900|-39000000|-39000000000|498355200000|1.11111111111111|89|089||Y
1|-40|-4000|-40000000|-40000000000|498441600000|1.0989010989011|90|090||Z
0|-41|-4100|-41000000|-41000000000|498528000000|1.08695652173913|91|091||[
1|-42|-4200|-42000000|-42000000000|498614400000|1.0752688172043|92|092||\
0|-43|-4300|-43000000|-43000000000|498700800000|1.06382978723404|93|093||]
1|-44|-4400|-44000000|-44000000000|498787200000|1.05263157894737|94|094||^
0|-45|-4500|-45000000|-45000000000|498873600000|1.04166666666667|95|095||_
1|-46|-4600|-46000000|-46000000000|498960000000|1.03092783505155|96|096||`
0|-47|-4700|-47000000|-47000000000|499046400000|1.02040816326531|97|097||a
1|-48|-4800|-48000000|-48000000000|499132800000|1.01010101010101|98|098||b
0|-49|-4900|-49000000|-49000000000|499219200000|1.0|99|099||c

3
tests/queries/003-cross-join-count.sql Normal file
View File

@ -0,0 +1,3 @@
100-rows-10.parquet
select count(*) from (select * from test t1, test t2);
10000

1
tests/test-all
View File

@ -6,3 +6,4 @@ here=$(dirname "${BASH_SOURCE[0]}")
set -x set -x
"$here"/test-unsupported "$here"/test-unsupported
"$here"/test-supported "$here"/test-supported
"$here"/test-queries

41
tests/test-queries Executable file
View File

@ -0,0 +1,41 @@
#!/bin/bash
set -euo pipefail
# For each files in tests/queries/*, mount that parquet file, run the query, and compare
# its output.
run_query() {
file=${1:?must provide testcase file}
parquet_file=${2:?must provide parquet file to load}
query=${3:?must provide query to run}
basename=$(basename "$file")
cat <<EOF
.load parquet/libparquet
.testcase $basename
.bail on
CREATE VIRTUAL TABLE test USING parquet('parquet-generator/$parquet_file');
$query
EOF
}
main() {
root=$(dirname "${BASH_SOURCE[0]}")/..
root=$(readlink -f "$root")
cd "$root"
queries=$(find tests/queries -type f -name '*.sql')
while read -r file; do
echo "Testing: $file"
parquet_file=$(head -n1 "$file")
query=$(head -n2 "$file" | tail -n1)
results=$(tail -n+3 "$file")
if ! "$root"/sqlite/sqlite3 -init <(run_query "$file" "$parquet_file" "$query") < /dev/null > /dev/null 2> testcase-err.txt; then
echo "...FAILED; check testcase-{out,err}.txt" >&2
exit 1
fi
diff testcase-out.txt <(echo "$results")
done < <(echo "$queries")
}
main "$@"