detokenize.h

// Copyright 2020 The Pigweed Authors
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
// use this file except in compliance with the License. You may obtain a copy of
// the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations under
// the License.
 
// This file provides the Detokenizer class, which is used to decode tokenized
// strings.  To use a Detokenizer, load a binary format token database into
// memory, construct a TokenDatabase, and pass it to a Detokenizer:
//
//   std::vector data = ReadFile("my_tokenized_strings.db");
//   Detokenizer detok(TokenDatabase::Create(data));
//
//   DetokenizedString result = detok.Detokenize(my_data);
//   std::cout << result.BestString() << '\n';
//
#pragma once
 
#include <cstddef>
#include <cstdint>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
 
#include "pw_result/result.h"
#include "pw_span/span.h"
#include "pw_stream/stream.h"
#include "pw_tokenizer/internal/decode.h"
#include "pw_tokenizer/token_database.h"
#include "pw_tokenizer/tokenize.h"
 
namespace pw::tokenizer {
 
 
class Detokenizer;
 
using TokenizedStringEntry = std::pair<FormatString, uint32_t /*date removed*/>;
using DomainTokenEntriesMap = std::unordered_map<
    std::string,
    std::unordered_map<uint32_t, std::vector<TokenizedStringEntry>>>;
 
class DetokenizedString {
 public:
  DetokenizedString(const Detokenizer& detokenizer,
                    bool recursion,
                    uint32_t token,
                    const span<const TokenizedStringEntry>& entries,
                    const span<const std::byte>& arguments);
 
  DetokenizedString() : has_token_(false), ok_(false) {}
 
  bool ok() const { return ok_; }
 
  const std::vector<DecodedFormatString>& matches() const { return matches_; }
 
  const uint32_t& token() const { return token_; }
 
  const std::string& BestString() const { return best_string_; }
 
  std::string BestStringWithErrors() const;
 
 private:
  uint32_t token_;
  std::string best_string_;
  bool has_token_;
  bool ok_;
  std::vector<DecodedFormatString> matches_;
};
 
class Detokenizer {
 public:
  explicit Detokenizer(const TokenDatabase& database);
 
  explicit Detokenizer(DomainTokenEntriesMap&& database)
      : database_(std::move(database)) {}
 
  static Result<Detokenizer> FromElfSection(span<const std::byte> elf_section);
 
  static Result<Detokenizer> FromElfSection(span<const uint8_t> elf_section) {
    return FromElfSection(as_bytes(elf_section));
  }
 
  static Result<Detokenizer> FromElfFile(stream::SeekableReader& stream);
 
  static Result<Detokenizer> FromCsv(std::string_view csv);
 
  DetokenizedString Detokenize(const span<const std::byte>& encoded,
                               std::string_view domain = kDefaultDomain) const {
    return Detokenize(encoded, domain, false);
  }
 
  DetokenizedString Detokenize(const span<const uint8_t>& encoded,
                               std::string_view domain = kDefaultDomain) const {
    return Detokenize(as_bytes(encoded), domain);
  }
 
  DetokenizedString Detokenize(std::string_view encoded,
                               std::string_view domain = kDefaultDomain) const {
    return Detokenize(encoded.data(), encoded.size(), domain);
  }
 
  DetokenizedString Detokenize(const void* encoded,
                               size_t size_bytes,
                               std::string_view domain = kDefaultDomain) const {
    return Detokenize(span(static_cast<const std::byte*>(encoded), size_bytes),
                      domain);
  }
 
  DetokenizedString RecursiveDetokenize(
      const span<const std::byte>& encoded,
      std::string_view domain = kDefaultDomain) const {
    return Detokenize(encoded, domain, true);
  }
 
  DetokenizedString RecursiveDetokenize(
      const span<const uint8_t>& encoded,
      std::string_view domain = kDefaultDomain) const {
    return RecursiveDetokenize(as_bytes(encoded), domain);
  }
 
  DetokenizedString RecursiveDetokenize(
      std::string_view encoded,
      std::string_view domain = kDefaultDomain) const {
    return RecursiveDetokenize(encoded.data(), encoded.size(), domain);
  }
 
  DetokenizedString RecursiveDetokenize(
      const void* encoded,
      size_t size_bytes,
      std::string_view domain = kDefaultDomain) const {
    return RecursiveDetokenize(
        span(static_cast<const std::byte*>(encoded), size_bytes), domain);
  }
 
  DetokenizedString DetokenizeBase64Message(std::string_view text) const;
 
  std::string DetokenizeText(std::string_view text) const {
    return DetokenizeTextRecursive(text, kMaxDecodePasses);
  }
 
  std::string DecodeOptionallyTokenizedData(
      span<const std::byte> optionally_tokenized_data) const;
 
  const DomainTokenEntriesMap& database() const { return database_; }
 
  span<const TokenizedStringEntry> DatabaseLookup(
      Token token, std::string_view domain) const;
 
 private:
  // 4 passes supports detokenizing two layers of nested messages with tokenized
  // domains (e.g. ${${bar}#ab12cd34}#00000012), without allowing a hypothetical
  // detokenization cycle to continue for too long.
  static constexpr unsigned kMaxDecodePasses = 4;
 
  std::string DetokenizeTextRecursive(std::string_view text,
                                      unsigned max_passes) const;
 
  DetokenizedString Detokenize(const span<const std::byte>& encoded,
                               std::string_view domain,
                               bool recursion) const;
 
  DomainTokenEntriesMap database_;
};
 
 
}  // namespace pw::tokenizer