token_database.h

// Copyright 2020 The Pigweed Authors
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
// use this file except in compliance with the License. You may obtain a copy of
// the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations under
// the License.
#pragma once
 
#include <array>
#include <cstddef>
#include <cstdint>
#include <iterator>
 
namespace pw::tokenizer {
 
 
class TokenDatabase {
 private:
  // Internal struct that describes how the underlying binary token database
  // stores entries. RawEntries generally should not be used directly. Instead,
  // use an Entry, which contains a pointer to the entry's string.
  struct RawEntry {
    uint32_t token;
    uint32_t date_removed;
  };
 
  static_assert(sizeof(RawEntry) == 8u);
 
  template <typename T>
  static constexpr uint32_t ReadUint32(const T* bytes) {
    return static_cast<uint32_t>(static_cast<uint8_t>(bytes[0]) |
                                 static_cast<uint8_t>(bytes[1]) << 8 |
                                 static_cast<uint8_t>(bytes[2]) << 16 |
                                 static_cast<uint8_t>(bytes[3]) << 24);
  }
 
 public:
  static constexpr uint32_t kDateRemovedNever = 0xFFFFFFFF;
 
  struct Entry {
    uint32_t token;
 
    uint32_t date_removed;
 
    const char* string;
  };
 
  class iterator {
   public:
    using difference_type = std::ptrdiff_t;
    using value_type = Entry;
    using pointer = const Entry*;
    using reference = const Entry&;
    using iterator_category = std::forward_iterator_tag;
 
    constexpr iterator() : entry_{}, raw_(nullptr) {}
 
    constexpr iterator(const iterator& other) = default;
    constexpr iterator& operator=(const iterator& other) = default;
 
    constexpr iterator& operator++() {
      raw_ += sizeof(RawEntry);
      ReadRawEntry();
      // Move string_ to the character beyond the next null terminator.
      while (*entry_.string++ != '\0') {
      }
      return *this;
    }
    constexpr iterator operator++(int) {
      iterator previous(*this);
      operator++();
      return previous;
    }
    constexpr bool operator==(const iterator& rhs) const {
      return raw_ == rhs.raw_;
    }
    constexpr bool operator!=(const iterator& rhs) const {
      return raw_ != rhs.raw_;
    }
 
    constexpr const Entry& operator*() const { return entry_; }
 
    constexpr const Entry* operator->() const { return &entry_; }
 
    constexpr difference_type operator-(const iterator& rhs) const {
      return (raw_ - rhs.raw_) / static_cast<difference_type>(sizeof(RawEntry));
    }
 
   private:
    friend class TokenDatabase;
 
    // Constructs a new iterator to a valid entry.
    constexpr iterator(const char* raw_entry, const char* string)
        : entry_{0, 0, string}, raw_{raw_entry} {
      if (raw_entry != string) {  // raw_entry == string if the DB is empty
        ReadRawEntry();
      }
    }
 
    explicit constexpr iterator(const char* end) : entry_{}, raw_(end) {}
 
    constexpr void ReadRawEntry() {
      entry_.token = ReadUint32(raw_);
      entry_.date_removed = ReadUint32(raw_ + sizeof(entry_.token));
    }
 
    Entry entry_;
    const char* raw_;
  };
 
  using value_type = Entry;
  using size_type = std::size_t;
  using difference_type = std::ptrdiff_t;
  using reference = value_type&;
  using const_reference = const value_type&;
  using pointer = const value_type*;
  using const_pointer = const value_type*;
  using const_iterator = iterator;
  using reverse_iterator = std::reverse_iterator<iterator>;
  using const_reverse_iterator = std::reverse_iterator<const_iterator>;
 
  class Entries {
   public:
    constexpr Entries(const iterator& begin, const iterator& end)
        : begin_(begin), end_(end) {}
 
    // The number of entries in this list.
    constexpr size_type size() const {
      return static_cast<size_type>(end_ - begin_);
    }
 
    // True of the list is empty.
    constexpr bool empty() const { return begin_ == end_; }
 
    // Accesses the specified entry in this set. The index must be less than
    // size(). This operation is O(n) in size().
    Entry operator[](size_type index) const;
 
    constexpr const iterator& begin() const { return begin_; }
    constexpr const iterator& end() const { return end_; }
 
   private:
    iterator begin_;
    iterator end_;
  };
 
  template <typename ByteArray>
  static constexpr bool IsValid(const ByteArray& bytes) {
    return HasValidHeader(bytes) && EachEntryHasAString(bytes);
  }
 
  template <const auto& kDatabaseBytes>
  static constexpr TokenDatabase Create() {
    static_assert(
        HasValidHeader<decltype(kDatabaseBytes)>(kDatabaseBytes),
        "Databases must start with a 16-byte header that begins with TOKENS.");
 
    static_assert(EachEntryHasAString<decltype(kDatabaseBytes)>(kDatabaseBytes),
                  "The database must have at least one string for each entry.");
 
    return TokenDatabase(std::data(kDatabaseBytes));
  }
 
  template <typename ByteArray>
  static constexpr TokenDatabase Create(const ByteArray& database_bytes) {
    return IsValid<ByteArray>(database_bytes)
               ? TokenDatabase(std::data(database_bytes))
               : TokenDatabase();  // Invalid database.
  }
  constexpr TokenDatabase() : begin_{.data = nullptr}, end_{.data = nullptr} {}
 
  Entries Find(uint32_t token) const;
 
  constexpr size_type size() const {
    return static_cast<size_type>(end_.data - begin_.data) / sizeof(RawEntry);
  }
 
  constexpr bool ok() const { return begin_.data != nullptr; }
 
  constexpr iterator begin() const { return iterator(begin_.data, end_.data); }
 
  constexpr iterator end() const { return iterator(end_.data); }
 
 private:
  struct Header {
    std::array<char, 6> magic;
    uint16_t version;
    uint32_t entry_count;
    uint32_t reserved;
  };
 
  static_assert(sizeof(Header) == 2 * sizeof(RawEntry));
 
  template <typename ByteArray>
  static constexpr bool HasValidHeader(const ByteArray& bytes) {
    static_assert(sizeof(*std::data(bytes)) == 1u);
 
    if (std::size(bytes) < sizeof(Header)) {
      return false;
    }
 
    // Check the magic number and version.
    for (size_type i = 0; i < kMagicAndVersion.size(); ++i) {
      if (bytes[i] != kMagicAndVersion[i]) {
        return false;
      }
    }
 
    return true;
  }
 
  template <typename ByteArray>
  static constexpr bool EachEntryHasAString(const ByteArray& bytes) {
    const size_type entries = ReadEntryCount(std::data(bytes));
 
    // Check that the data is large enough to have a string table.
    if (std::size(bytes) < StringTable(entries)) {
      return false;
    }
 
    // Count the strings in the string table.
    size_type string_count = 0;
    for (auto i =
             std::begin(bytes) + static_cast<ptrdiff_t>(StringTable(entries));
         i < std::end(bytes);
         ++i) {
      string_count += (*i == '\0') ? 1 : 0;
    }
 
    // Check that there is at least one string for each entry.
    return string_count >= entries;
  }
 
  // Reads the number of entries from a database header. Cast to the bytes to
  // uint8_t to avoid sign extension if T is signed.
  template <typename T>
  static constexpr uint32_t ReadEntryCount(const T* header_bytes) {
    const T* bytes = header_bytes + offsetof(Header, entry_count);
    return ReadUint32(bytes);
  }
 
  // Calculates the offset of the string table.
  static constexpr size_type StringTable(size_type entries) {
    return sizeof(Header) + entries * sizeof(RawEntry);
  }
 
  // The magic number that starts the table is "TOKENS". The version is encoded
  // next as two bytes.
  static constexpr std::array<char, 8> kMagicAndVersion = {
      'T', 'O', 'K', 'E', 'N', 'S', '\0', '\0'};
 
  template <typename Byte>
  constexpr TokenDatabase(const Byte bytes[])
      : TokenDatabase(bytes + sizeof(Header),
                      bytes + StringTable(ReadEntryCount(bytes))) {
    static_assert(sizeof(Byte) == 1u);
  }
 
  // It is illegal to reinterpret_cast in constexpr functions, but acceptable to
  // use unions. Instead of using a reinterpret_cast to change the byte pointer
  // to a RawEntry pointer, have a separate overload for each byte pointer type
  // and store them in a union.
  constexpr TokenDatabase(const char* begin, const char* end)
      : begin_{.data = begin}, end_{.data = end} {}
 
  constexpr TokenDatabase(const unsigned char* begin, const unsigned char* end)
      : begin_{.unsigned_data = begin}, end_{.unsigned_data = end} {}
 
  constexpr TokenDatabase(const signed char* begin, const signed char* end)
      : begin_{.signed_data = begin}, end_{.signed_data = end} {}
 
  // Store the beginning and end pointers as a union to avoid breaking constexpr
  // rules for reinterpret_cast.
  union {
    const char* data;
    const unsigned char* unsigned_data;
    const signed char* signed_data;
  } begin_, end_;
};
 
 
}  // namespace pw::tokenizer