sqlite-parquet-vtable/parquet/parquet_table.cc

#include "parquet_table.h"

#include "parquet/api/reader.h"

ParquetTable::ParquetTable(std::string file, std::string tableName): file(file), tableName(tableName) {
  std::unique_ptr<parquet::ParquetFileReader> reader = parquet::ParquetFileReader::OpenFile(file.data());
  metadata = reader->metadata();
}

std::string ParquetTable::columnName(int i) {
  if(i == -1)
    return "rowid";
  return columnNames[i];
}

unsigned int ParquetTable::getNumColumns() {
  return columnNames.size();
}


std::string ParquetTable::CreateStatement() {
  std::unique_ptr<parquet::ParquetFileReader> reader = parquet::ParquetFileReader::OpenFile(
      file.data(),
      true,
      parquet::default_reader_properties(),
      metadata);
  std::string text("CREATE TABLE x(");
  auto schema = reader->metadata()->schema();

  for(auto i = 0; i < schema->num_columns(); i++) {
    auto _col = schema->GetColumnRoot(i);
    columnNames.push_back(_col->name());
  }

  for(auto i = 0; i < schema->num_columns(); i++) {
    auto _col = schema->GetColumnRoot(i);

    if(!_col->is_primitive()) {
      std::ostringstream ss;
      ss << __FILE__ << ":" << __LINE__ << ": column " << i << " has non-primitive type";
      throw std::invalid_argument(ss.str());
    }

    if(_col->is_repeated()) {
      std::ostringstream ss;
      ss << __FILE__ << ":" << __LINE__ << ": column " << i << " has non-scalar type";
      throw std::invalid_argument(ss.str());
    }

    parquet::schema::PrimitiveNode* col = (parquet::schema::PrimitiveNode*)_col;

    if(i > 0)
      text += ", ";

    text += "\"";
    // Horrifically inefficient, but easy to understand.
    std::string colName = col->name();
    for(char& c : colName) {
      if(c == '"')
        text += "\"\"";
      else
        text += c;
    }
    text += "\"";

    std::string type;

    parquet::Type::type physical = col->physical_type();
    parquet::ConvertedType::type converted = col->converted_type();
    // Be explicit about which types we understand so we don't mislead someone
    // whose unsigned ints start getting interpreted as signed. (We could
    // support this for UINT_8/16/32 -- and for UINT_64 we could throw if
    // the high bit was set.)
    if(converted == parquet::ConvertedType::NONE ||
        converted == parquet::ConvertedType::UTF8 ||
        converted == parquet::ConvertedType::DATE ||
        converted == parquet::ConvertedType::TIME_MILLIS ||
        converted == parquet::ConvertedType::TIMESTAMP_MILLIS ||
        converted == parquet::ConvertedType::TIME_MICROS ||
        converted == parquet::ConvertedType::TIMESTAMP_MICROS ||
        converted == parquet::ConvertedType::INT_8 ||
        converted == parquet::ConvertedType::INT_16 ||
        converted == parquet::ConvertedType::INT_32 ||
        converted == parquet::ConvertedType::INT_64) {
      switch(physical) {
        case parquet::Type::BOOLEAN:
          type = "TINYINT";
          break;
        case parquet::Type::INT32:
          if(converted == parquet::ConvertedType::NONE ||
              converted == parquet::ConvertedType::INT_32) {
            type = "INT";
          } else if(converted == parquet::ConvertedType::INT_8) {
            type = "TINYINT";
          } else if(converted == parquet::ConvertedType::INT_16) {
            type = "SMALLINT";
          }
          break;
        case parquet::Type::INT96:
          // INT96 is used for nanosecond precision on timestamps; we truncate
          // to millisecond precision.
        case parquet::Type::INT64:
          type = "BIGINT";
          break;
        case parquet::Type::FLOAT:
          type = "REAL";
          break;
        case parquet::Type::DOUBLE:
          type = "DOUBLE";
          break;
        case parquet::Type::BYTE_ARRAY:
          if(converted == parquet::ConvertedType::UTF8) {
            type = "TEXT";
          } else {
            type = "BLOB";
          }
          break;
        case parquet::Type::FIXED_LEN_BYTE_ARRAY:
          type = "BLOB";
          break;
        default:
          break;
      }
    }

    if(type.empty()) {
      std::ostringstream ss;
      ss << __FILE__ << ":" << __LINE__ << ": column " << i << " has unsupported type: " <<
        parquet::TypeToString(physical) << "/" << parquet::ConvertedTypeToString(converted);

      throw std::invalid_argument(ss.str());
    }

#ifdef DEBUG
    printf("col %d[name=%s, p=%d:%s, l=%d:%s] is %s\n",
        i,
        col->name().data(),
        col->physical_type(),
        parquet::TypeToString(col->physical_type()).data(),
        col->converted_type(),
        parquet::ConvertedTypeToString(col->converted_type()).data(),
        type.data());
#endif

    text += " ";
    text += type;
  }
  text +=");";
  return text;
}

std::shared_ptr<parquet::FileMetaData> ParquetTable::getMetadata() { return metadata; }

const std::string& ParquetTable::getFile() { return file; }
const std::string& ParquetTable::getTableName() { return tableName; }