metadata.hpp

Go to the documentation of this file.

// SPDX-License-Identifier: MIT
 
#ifndef LIBZARR_METADATA_HPP
#define LIBZARR_METADATA_HPP
 
#include <cmath>
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <limits>
#include <optional>
#include <string>
#include <vector>
 
#include <nlohmann/json.hpp>
 
#include "libzarr/detail/common.hpp"
#include "libzarr/types.hpp"
 
 
namespace zarr {
 
using json = nlohmann::json;
 
struct CodecSpec {
  std::string name;
  json configuration = json::object();
};
 
namespace codec {
 
[[nodiscard]] inline CodecSpec gzip(int level = 5) { return {"gzip", {{"level", level}}}; }
 
[[nodiscard]] inline CodecSpec zlib(int level = 5) { return {"zlib", {{"level", level}}}; }
 
[[nodiscard]] inline CodecSpec blosc(const std::string& cname = "lz4", int clevel = 5,
                                     const std::string& shuffle = "shuffle") {
  return {"blosc", {{"cname", cname}, {"clevel", clevel}, {"shuffle", shuffle}}};
}
 
[[nodiscard]] inline CodecSpec zstd(int level = 0, bool checksum = false) {
  return {"zstd", {{"level", level}, {"checksum", checksum}}};
}
 
[[nodiscard]] inline CodecSpec crc32c() { return {"crc32c", {}}; }
 
[[nodiscard]] inline CodecSpec shuffle(int elementsize = 0) {
  return {"shuffle", {{"elementsize", elementsize}}};
}
 
}  // namespace codec
 
enum class ChunkKeyKind : std::uint8_t {
  v2,
  v3_default,
};
 
struct ShardLevel {
  std::vector<std::uint64_t> shard_shape;
  std::vector<CodecSpec> index_codecs;
  bool index_at_end = true;
};
 
struct ArrayMeta {
  ZarrFormat format = ZarrFormat::v2;
  std::vector<std::uint64_t> shape;
  std::vector<std::uint64_t> chunk_shape;
  DataType dtype;
  std::optional<Bytes> fill;
  ChunkKeyKind key_encoding = ChunkKeyKind::v2;
  char dimension_separator = '.';
  json dimension_names;
  std::vector<CodecSpec> codecs;
  std::vector<ShardLevel> shard_levels;
  json attributes = json::object();
 
  [[nodiscard]] std::uint64_t element_count() const {
    return detail::checked_product(shape, "array shape");
  }
  [[nodiscard]] std::uint64_t chunk_element_count() const {
    return detail::checked_product(chunk_shape, "chunk shape");
  }
  [[nodiscard]] std::vector<std::uint64_t> grid_shape() const {
    std::vector<std::uint64_t> grid(shape.size());
    for (std::size_t d = 0; d < shape.size(); ++d) {
      grid[d] = detail::ceil_div(shape[d], chunk_shape[d]);
    }
    return grid;
  }
};
 
struct OpenOptions {
  bool lenient = false;
};
 
[[nodiscard]] inline Bytes canonical_json_bytes(const json& j) {
  const std::string text = j.dump(4);
  return {text.begin(), text.end()};
}
 
namespace detail {
 
template <typename Fn>
auto guard_json(const std::string& ctx, const Fn& fn) -> decltype(fn()) {
  try {
    return fn();
  } catch (const json::exception& e) {
    throw error(ctx + ": malformed metadata (" + e.what() + ")");
  }
}
 
inline json parse_json(const Bytes& bytes, const std::string& ctx) {
  try {
    return json::parse(bytes.begin(), bytes.end());
  } catch (const json::exception& e) {
    throw error(ctx + ": " + e.what());
  }
}
 
inline std::uint64_t json_to_uint64(const json& v, const std::string& ctx) {
  if (v.is_number_unsigned()) {
    return v.get<std::uint64_t>();
  }
  if (v.is_number_integer()) {
    const auto i = v.get<std::int64_t>();
    if (i < 0) {
      throw error(ctx + ": expected a non-negative integer, got " + std::to_string(i));
    }
    return static_cast<std::uint64_t>(i);
  }
  throw error(ctx + ": expected a non-negative integer, got " + v.dump());
}
 
inline std::int64_t lenient_int(const json& obj, const char* key, std::int64_t fallback,
                                const std::string& ctx) {
  const auto it = obj.find(key);
  if (it == obj.end()) {
    return fallback;
  }
  if (it->is_number_integer() || it->is_number_unsigned()) {
    return it->get<std::int64_t>();
  }
  if (it->is_string()) {
    const auto s = it->get<std::string>();
    if (!s.empty()) {
      char* end = nullptr;
      const long long v = std::strtoll(s.c_str(), &end, 10);
      if (end == s.c_str() + s.size()) {
        return static_cast<std::int64_t>(v);
      }
    }
  }
  throw error(ctx + ": '" + key + "' must be an integer, got " + it->dump());
}
 
template <typename T>
Bytes scalar_bytes(T value) {
  Bytes out(sizeof(T));
  std::memcpy(out.data(), &value, sizeof(T));
  return out;
}
 
inline std::vector<std::uint64_t> parse_extents(const json& v, const char* name,
                                                const std::string& ctx) {
  if (!v.is_array()) {
    throw error(ctx + ": '" + name + "' must be an array");
  }
  std::vector<std::uint64_t> out;
  out.reserve(v.size());
  for (const json& e : v) {
    out.push_back(json_to_uint64(e, ctx + ": " + name));
  }
  return out;
}
 
inline Bytes quiet_nan_bytes(DType kind) {
  if (kind == DType::float16) {
    return scalar_bytes<std::uint16_t>(0x7e00U);
  }
  if (kind == DType::float32) {
    return scalar_bytes<std::uint32_t>(0x7fc00000U);
  }
  assert(kind == DType::float64);
  return scalar_bytes<std::uint64_t>(0x7ff8000000000000ULL);
}
 
inline Bytes infinity_bytes(DType kind, bool negative) {
  if (kind == DType::float16) {
    return scalar_bytes<std::uint16_t>(negative ? 0xfc00U : 0x7c00U);
  }
  if (kind == DType::float32) {
    const float inf = std::numeric_limits<float>::infinity();
    return scalar_bytes(negative ? -inf : inf);
  }
  assert(kind == DType::float64);
  const double inf = std::numeric_limits<double>::infinity();
  return scalar_bytes(negative ? -inf : inf);
}
 
inline Bytes fill_from_double(double value, DataType dt, const std::string& ctx);
 
inline Bytes fill_from_int(std::int64_t value, DataType dt, const std::string& ctx) {
  const auto check = [&](std::int64_t lo, std::int64_t hi) {
    if (value < lo || value > hi) {
      throw error(ctx + ": fill_value " + std::to_string(value) + " out of range for dtype");
    }
  };
  switch (dt.kind) {
    case DType::boolean:
      check(0, 1);
      return scalar_bytes<std::uint8_t>(static_cast<std::uint8_t>(value));
    case DType::int8:
      check(std::numeric_limits<std::int8_t>::min(), std::numeric_limits<std::int8_t>::max());
      return scalar_bytes<std::int8_t>(static_cast<std::int8_t>(value));
    case DType::int16:
      check(std::numeric_limits<std::int16_t>::min(), std::numeric_limits<std::int16_t>::max());
      return scalar_bytes<std::int16_t>(static_cast<std::int16_t>(value));
    case DType::int32:
      check(std::numeric_limits<std::int32_t>::min(), std::numeric_limits<std::int32_t>::max());
      return scalar_bytes<std::int32_t>(static_cast<std::int32_t>(value));
    case DType::int64:
      return scalar_bytes<std::int64_t>(value);
    case DType::uint8:
    case DType::uint16:
    case DType::uint32:
    case DType::uint64: {
      if (value < 0) {
        throw error(ctx + ": fill_value " + std::to_string(value) +
                    " is negative for unsigned dtype");
      }
      const auto u = static_cast<std::uint64_t>(value);
      if (dt.kind == DType::uint8 && u > std::numeric_limits<std::uint8_t>::max()) {
        check(0, 255);
      }
      if (dt.kind == DType::uint16 && u > std::numeric_limits<std::uint16_t>::max()) {
        check(0, 65535);
      }
      if (dt.kind == DType::uint32 && u > std::numeric_limits<std::uint32_t>::max()) {
        check(0, 4294967295LL);
      }
      if (dt.kind == DType::uint8) {
        return scalar_bytes<std::uint8_t>(static_cast<std::uint8_t>(u));
      }
      if (dt.kind == DType::uint16) {
        return scalar_bytes<std::uint16_t>(static_cast<std::uint16_t>(u));
      }
      if (dt.kind == DType::uint32) {
        return scalar_bytes<std::uint32_t>(static_cast<std::uint32_t>(u));
      }
      return scalar_bytes<std::uint64_t>(u);
    }
    case DType::float32:
    case DType::float64:
      return fill_from_double(static_cast<double>(value), dt, ctx);
    default:
      throw error(ctx + ": numeric fill_value invalid for this dtype");
  }
}
 
inline Bytes fill_from_uint(std::uint64_t value, DataType dt, const std::string& ctx) {
  if (value <= static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max())) {
    return fill_from_int(static_cast<std::int64_t>(value), dt, ctx);
  }
  if (dt.kind != DType::uint64) {
    throw error(ctx + ": fill_value " + std::to_string(value) + " out of range for dtype");
  }
  return scalar_bytes<std::uint64_t>(value);
}
 
inline Bytes fill_from_double(double value, DataType dt, const std::string& ctx) {
  switch (dt.kind) {
    case DType::float16:
      return scalar_bytes<std::uint16_t>(double_to_half_bits(value));
    case DType::float32:
      return scalar_bytes<float>(static_cast<float>(value));
    case DType::float64:
      return scalar_bytes<double>(value);
    default:
      // Tolerance: integral fills occasionally arrive as JSON floats
      // (e.g. 1.0 for an int dtype); accept when exactly integral.
      if (std::nearbyint(value) == value &&
          value >= static_cast<double>(std::numeric_limits<std::int64_t>::min()) &&
          value <= static_cast<double>(std::numeric_limits<std::int64_t>::max())) {
        return fill_from_int(static_cast<std::int64_t>(value), dt, ctx);
      }
      throw error(ctx + ": non-integral fill_value for integer dtype");
  }
}
 
inline json fill_to_json(const std::optional<Bytes>& fill, DataType dt) {
  if (!fill) {
    return nullptr;
  }
  const std::uint8_t* p = fill->data();
  const auto load = [&](auto probe) {
    decltype(probe) v;
    std::memcpy(&v, p, sizeof(v));
    return v;
  };
  switch (dt.kind) {
    case DType::boolean:
      return load(std::uint8_t{}) != 0;
    case DType::int8:
      return load(std::int8_t{});
    case DType::int16:
      return load(std::int16_t{});
    case DType::int32:
      return load(std::int32_t{});
    case DType::int64:
      return load(std::int64_t{});
    case DType::uint8:
      return load(std::uint8_t{});
    case DType::uint16:
      return load(std::uint16_t{});
    case DType::uint32:
      return load(std::uint32_t{});
    case DType::uint64:
      return load(std::uint64_t{});
    case DType::float16:
    case DType::float32:
    case DType::float64: {
      double v = 0;
      if (dt.kind == DType::float16) {
        v = half_bits_to_double(load(std::uint16_t{}));
      } else if (dt.kind == DType::float32) {
        v = static_cast<double>(load(float{}));
      } else {
        v = load(double{});
      }
      if (std::isnan(v)) {
        return "NaN";
      }
      if (std::isinf(v)) {
        return v > 0 ? "Infinity" : "-Infinity";
      }
      return v;
    }
    case DType::complex64:
    case DType::complex128: {
      const DType component = dt.kind == DType::complex64 ? DType::float32 : DType::float64;
      const std::uint32_t half = dt.itemsize / 2;
      return json::array({fill_to_json(Bytes(p, p + half), DataType::of(component)),
                          fill_to_json(Bytes(p + half, p + dt.itemsize), DataType::of(component))});
    }
    case DType::raw:
      return base64_encode(p, dt.itemsize);
    default:
      throw error("fill_value emission not implemented for this dtype");
  }
}
 
}  // namespace detail
 
}  // namespace zarr
 
#endif  // LIBZARR_METADATA_HPP