Tokenization

Oveview

Convert string literals to tokens.

Classes
class	pw::tokenizer::EncodedMessage< kMaxSizeBytes >

Macros
#define	PW_TOKENIZE_ENUM(fully_qualified_name, ...)
#define	PW_TOKENIZE_ENUM_CUSTOM(fully_qualified_name, ...)
#define	PW_TOKENIZER_NESTED_PREFIX   PW_TOKENIZER_NESTED_PREFIX_STR[0]
#define	PW_TOKEN_FMT(...)   PW_DELEGATE_BY_ARG_COUNT(_PW_TOKEN_FMT_, __VA_ARGS__)
	Format specifier for a token argument.
#define	PW_NESTED_TOKEN_FMT(...)    PW_DELEGATE_BY_ARG_COUNT(_PW_NESTED_TOKEN_FMT_, __VA_ARGS__)
#define	PW_TOKENIZER_DEFAULT_DOMAIN   ""
#define	PW_TOKENIZE_STRING(...)    PW_DELEGATE_BY_ARG_COUNT(_PW_TOKENIZE_STRING_, __VA_ARGS__)
#define	PW_TOKENIZE_STRING_EXPR(...)    PW_DELEGATE_BY_ARG_COUNT(_PW_TOKENIZE_STRING_EXPR_, __VA_ARGS__)
#define	PW_TOKENIZE_STRING_DOMAIN(domain, string_literal)    PW_TOKENIZE_STRING_MASK(domain, UINT32_MAX, string_literal)
#define	PW_TOKENIZE_STRING_DOMAIN_EXPR(domain, string_literal)
#define	PW_TOKENIZE_STRING_MASK(domain, mask, string_literal)
#define	PW_TOKENIZE_STRING_MASK_EXPR(domain, mask, string_literal)
#define	PW_TOKENIZE_TO_BUFFER(buffer, buffer_size_pointer, format, ...)
#define	PW_TOKENIZE_TO_BUFFER_DOMAIN( domain, buffer, buffer_size_pointer, format, ...)
#define	PW_TOKENIZE_TO_BUFFER_MASK( domain, mask, buffer, buffer_size_pointer, format, ...)
#define	PW_TOKENIZER_REPLACE_FORMAT_STRING(...)    _PW_TOKENIZER_REPLACE_FORMAT_STRING(PW_EMPTY_ARGS(__VA_ARGS__), __VA_ARGS__)
	Low-level macro for calling functions that handle tokenized strings.
#define	PW_TOKENIZER_ARG_TYPES(...)    PW_DELEGATE_BY_ARG_COUNT(_PW_TOKENIZER_TYPES_, __VA_ARGS__)
#define	PW_TOKENIZE_FORMAT_STRING(domain, mask, format, ...)
#define	PW_TOKENIZE_FORMAT_STRING_ANY_ARG_COUNT(domain, mask, format, ...)
#define	PW_TOKENIZER_DEFINE_TOKEN(token, domain, string)

Typedefs
typedef uint32_t	pw_tokenizer_Token

Functions
size_t	pw_tokenizer_PrefixedBase64Encode (const void *binary_message, size_t binary_size_bytes, void *output_buffer, size_t output_buffer_size_bytes)
size_t	pw_tokenizer_PrefixedBase64Decode (const void *base64_message, size_t base64_size_bytes, void *output_buffer, size_t output_buffer_size)
constexpr size_t	pw::tokenizer::Base64EncodedStringSize (size_t message_size)
constexpr size_t	pw::tokenizer::Base64EncodedBufferSize (size_t message_size)
size_t	pw::tokenizer::PrefixedBase64Encode (span< const std::byte > binary_message, span< char > output_buffer)
size_t	pw::tokenizer::PrefixedBase64Encode (span< const uint8_t > binary_message, span< char > output_buffer)
	Also accept a span<const uint8_t> for the binary message.
void	pw::tokenizer::PrefixedBase64Encode (span< const std::byte > binary_message, InlineString<> &output)
void	pw::tokenizer::PrefixedBase64Encode (span< const uint8_t > binary_message, InlineString<> &output)
template<size_t kMaxBinaryMessageSizeBytes>
auto	pw::tokenizer::PrefixedBase64Encode (span< const std::byte > binary_message)
template<size_t kMaxBinaryMessageSizeBytes>
auto	pw::tokenizer::PrefixedBase64Encode (span< const uint8_t > binary_message)
size_t	pw::tokenizer::PrefixedBase64Decode (std::string_view base64_message, span< std::byte > output_buffer)
size_t	pw::tokenizer::PrefixedBase64DecodeInPlace (span< std::byte > buffer)
size_t	pw::tokenizer::PrefixedBase64DecodeInPlace (span< char > buffer)
template<typename CharT >
void	pw::tokenizer::PrefixedBase64DecodeInPlace (InlineBasicString< CharT > &string)
template<typename... ArgTypes>
constexpr size_t	pw::tokenizer::MinEncodingBufferSizeBytes ()
size_t	pw::tokenizer::EncodeArgs (pw_tokenizer_ArgTypes types, va_list args, span< std::byte > output)
size_t	pw_tokenizer_EncodeArgs (pw_tokenizer_ArgTypes types, va_list args, void *output_buffer, size_t output_buffer_size)
static size_t	pw_tokenizer_EncodeInt (int value, void *output, size_t output_size_bytes)
static size_t	pw_tokenizer_EncodeInt64 (int64_t value, void *output, size_t output_size_bytes)
template<typename T >
constexpr auto	pw::tokenizer::EnumToToken (T value)
	Tokenizes a given enumerator value. Used in the vase of a tokenizing log backend.
template<typename T >
constexpr const char *	pw::tokenizer::EnumToString (T value)
	Returns a string representation of a given enumerator value name. Used in the case of a non-tokenizing log backend.

Macro Definition Documentation

PW_NESTED_TOKEN_FMT

#define PW_NESTED_TOKEN_FMT

(

...

)

PW_DELEGATE_BY_ARG_COUNT(_PW_NESTED_TOKEN_FMT_, __VA_ARGS__)

Format specifier for a doubly-nested token argument. Doubly-nested token arguments are useful when the domain and/or token may not be known at the time of logging. For example, if an external function is required to return a domain and token for logging, PW_NESTED_TOKEN_FMT still allows for the value to be logged as it tokenizes the domain value as well. It can either take an argument of a domain value, or no argument at all if there is no specified domain.

PW_NESTED_TOKEN_FMT() expands to ${$#x}#%08x PW_NESTED_TOKEN_FMT(domain_value) expands to ${${domain_value}#x}#%08x

An example of its application could look similar to this:

std::pair<PW_LOG_TOKEN_TYPE, PW_LOG_TOKEN_TYPE> GetDomainAndToken(...) {...}
 
const auto [domain, token] = GetDomainAndToken(...);
PW_LOG("Nested Token " PW_NESTED_TOKEN_FMT("enum_domain"), domain, token);

PW_TOKENIZE_ENUM

#define PW_TOKENIZE_ENUM	(		fully_qualified_name,
			...
	)

Value:

  PW_APPLY(_PW_TOKENIZE_ENUMERATOR,                               \
           _PW_SEMICOLON,                                         \
           fully_qualified_name,                                  \
           __VA_ARGS__);                                          \
  [[maybe_unused]] constexpr const char* PwTokenizerEnumToString( \
      fully_qualified_name _pw_enum_value) {                      \
    switch (_pw_enum_value) {                                     \
      PW_APPLY(_PW_TOKENIZE_TO_STRING_CASE,                       \
               _PW_SEMICOLON,                                     \
               fully_qualified_name,                              \
               __VA_ARGS__);                                      \
    }                                                             \
    return "Unknown " #fully_qualified_name " value";             \
  }                                                               \
  static_assert(true)

Tokenizes the given values within an enumerator. All values of the enumerator must be present to compile and have the enumerator be tokenized successfully. This macro should be in the same namespace as the enum declaration to use the pw::tokenizer::EnumToString function and avoid compilation errors.

PW_TOKENIZE_ENUM_CUSTOM

#define PW_TOKENIZE_ENUM_CUSTOM	(		fully_qualified_name,
			...
	)

Value:

  PW_APPLY(_PW_TOKENIZE_ENUMERATOR_CUSTOM,                        \
           _PW_SEMICOLON,                                         \
           fully_qualified_name,                                  \
           __VA_ARGS__);                                          \
  [[maybe_unused]] constexpr const char* PwTokenizerEnumToString( \
      fully_qualified_name _pw_enum_value) {                      \
    switch (_pw_enum_value) {                                     \
      PW_APPLY(_PW_TOKENIZE_TO_STRING_CASE_CUSTOM,                \
               _PW_SEMICOLON,                                     \
               fully_qualified_name,                              \
               __VA_ARGS__);                                      \
    }                                                             \
    return "Unknown " #fully_qualified_name " value";             \
  }                                                               \
  static_assert(true)

Tokenizes a custom string for each given values within an enumerator. All values of the enumerator must be followed by a custom string as a tuple (value, "string"). All values of the enumerator (and their associated custom string) must be present to compile and have the custom strings be tokenized successfully. This macro should be in the same namespace as the enum declaration to use the pw::tokenizer::EnumToString function and avoid compilation errors.

PW_TOKENIZE_FORMAT_STRING

#define PW_TOKENIZE_FORMAT_STRING	(		domain,
			mask,
			format,
			...
	)

Value:

  static_assert(                                                               \
      PW_FUNCTION_ARG_COUNT(__VA_ARGS__) <= PW_TOKENIZER_MAX_SUPPORTED_ARGS,   \
      "Tokenized strings cannot have more than "                               \
      PW_STRINGIFY(PW_TOKENIZER_MAX_SUPPORTED_ARGS) " arguments; "             \
      PW_STRINGIFY(PW_FUNCTION_ARG_COUNT(__VA_ARGS__))                         \
      " arguments were used for " #format " (" #__VA_ARGS__ ")"); \
  PW_TOKENIZE_FORMAT_STRING_ANY_ARG_COUNT(domain, mask, format, __VA_ARGS__)

Tokenizes a format string with optional arguments and sets the _pw_tokenizer_token variable to the token. Must be used in its own scope, since the same variable is used in every invocation.

The tokenized string uses the specified domain. Use PW_TOKENIZER_DEFAULT_DOMAIN for the default. The token also may be masked; use UINT32_MAX to keep all bits.

This macro checks that the printf-style format string matches the arguments and that no more than PW_TOKENIZER_MAX_SUPPORTED_ARGS are provided. It then stores the format string in a special section, and calculates the string's token at compile time.

PW_TOKENIZE_FORMAT_STRING_ANY_ARG_COUNT

#define PW_TOKENIZE_FORMAT_STRING_ANY_ARG_COUNT	(		domain,
			mask,
			format,
			...
	)

Value:

  if (0) { /* Do not execute to prevent double evaluation of the arguments. */ \
    pw_tokenizer_CheckFormatString(format PW_COMMA_ARGS(__VA_ARGS__));         \
  }                                                                            \
                                                                               \
  _PW_TOKENIZE_VALIDATE_FORMAT_STRING(format);                                 \
                                                                               \
  /* Tokenize the string to a pw_tokenizer_Token at compile time. */           \
  static _PW_TOKENIZER_CONST pw_tokenizer_Token _pw_tokenizer_token =          \
      _PW_TOKENIZER_MASK_TOKEN(mask, format);                                  \
                                                                               \
  PW_TOKENIZER_DEFINE_TOKEN(_pw_tokenizer_token, domain, format)

Equivalent to PW_TOKENIZE_FORMAT_STRING, but supports any number of arguments.

This is a low-level macro that should rarely be used directly. It is intended for situations when pw_tokenizer_ArgTypes is not used. There are two situations where pw_tokenizer_ArgTypes is unnecessary:

• The exact format string argument types and count are fixed.
• The format string supports a variable number of arguments of only one type. In this case, PW_FUNCTION_ARG_COUNT may be used to pass the argument count to the function.

NOTE: The %.*s format specifier cannot be used. See https://pwbug.dev/408040194 for details.

PW_TOKENIZE_STRING

#define PW_TOKENIZE_STRING

(

...

)

PW_DELEGATE_BY_ARG_COUNT(_PW_TOKENIZE_STRING_, __VA_ARGS__)

Converts a string literal to a pw_tokenizer_Token (uint32_t) token in a standalone statement. C and C++ compatible. In C++, the string may be a literal or a constexpr char array, including function variables like __func__. In C, the argument must be a string literal. In either case, the string must be null terminated, but may contain any characters (including '\0').

// Tokenizes a string literal in the given domain.
constexpr uint32_t token = PW_TOKENIZE_STRING("domain", "string literal");
 
// Tokenizes a string literal in the default domain ("").
constexpr uint32_t other = PW_TOKENIZE_STRING("string literal");

PW_TOKENIZE_STRING_DOMAIN

#define PW_TOKENIZE_STRING_DOMAIN	(		domain,
			string_literal
	)		PW_TOKENIZE_STRING_MASK(domain, UINT32_MAX, string_literal)

Tokenizes a string literal in a standalone statement using the specified domain. C and C++ compatible.

PW_TOKENIZE_STRING_DOMAIN_EXPR

#define PW_TOKENIZE_STRING_DOMAIN_EXPR	(		domain,
			string_literal
	)

Value:

  [&] {                                                        \
    constexpr uint32_t lambda_ret_token =                      \
        PW_TOKENIZE_STRING_DOMAIN(domain, string_literal);     \
    return lambda_ret_token;                                   \
  }()

Tokenizes a string literal using the specified domain within an expression. Requires C++.

PW_TOKENIZE_STRING_EXPR

#define PW_TOKENIZE_STRING_EXPR

(

...

)

PW_DELEGATE_BY_ARG_COUNT(_PW_TOKENIZE_STRING_EXPR_, __VA_ARGS__)

Converts a string literal to a uint32_t token within an expression. Requires C++.

// Tokenizes a string literal in the given domain within an expression.
DoSomethingWithToken(PW_TOKENIZE_STRING_EXPR("domain", "Succeed"));
 
// Tokenizes a string literal in the default domain within an expression.
DoSomethingWithToken(PW_TOKENIZE_STRING_EXPR("Succeed"));

PW_TOKENIZE_STRING_MASK

#define PW_TOKENIZE_STRING_MASK	(		domain,
			mask,
			string_literal
	)

Value:

  /* assign to a variable */ _PW_TOKENIZER_MASK_TOKEN(mask, string_literal); \
                                                                             \
  static_assert(0 < (mask) && (mask) <= UINT32_MAX,                          \
                "Tokenizer masks must be non-zero uint32_t values.");        \
                                                                             \
  PW_TOKENIZER_DEFINE_TOKEN(                                                 \
      _PW_TOKENIZER_MASK_TOKEN(mask, string_literal), domain, string_literal)

Tokenizes a string literal in a standalone statement using the specified domain and bit mask. C and C++ compatible.

PW_TOKENIZE_STRING_MASK_EXPR

#define PW_TOKENIZE_STRING_MASK_EXPR	(		domain,
			mask,
			string_literal
	)

Value:

  [&] {                                                            \
    constexpr uint32_t lambda_ret_token =                          \
        PW_TOKENIZE_STRING_MASK(domain, mask, string_literal);     \
    return lambda_ret_token;                                       \
  }()

Tokenizes a string literal using the specified domain and bit mask within an expression. Requires C++.

PW_TOKENIZE_TO_BUFFER

#define PW_TOKENIZE_TO_BUFFER	(		buffer,
			buffer_size_pointer,
			format,
			...
	)

Value:

  PW_TOKENIZE_TO_BUFFER_DOMAIN(PW_TOKENIZER_DEFAULT_DOMAIN,             \
                               buffer,                                  \
                               buffer_size_pointer,                     \
                               format,                                  \
                               __VA_ARGS__)

Encodes a tokenized string and arguments to the provided buffer. The size of the buffer is passed via a pointer to a size_t. After encoding is complete, the size_t is set to the number of bytes written to the buffer.

The macro's arguments are equivalent to the following function signature:

TokenizeToBuffer(void* buffer,
                 size_t* buffer_size_pointer,
                 const char* format,
                 ...);  // printf-style arguments

For example, the following encodes a tokenized string with a temperature to a buffer. The buffer is passed to a function to send the message over a UART.

uint8_t buffer[32];
size_t size_bytes = sizeof(buffer);
PW_TOKENIZE_TO_BUFFER(
    buffer, &size_bytes, "Temperature (C): %0.2f", temperature_c);
MyProject_EnqueueMessageForUart(buffer, size);

While PW_TOKENIZE_TO_BUFFER is very flexible, it must be passed a buffer, which increases its code size footprint at the call site.

PW_TOKENIZE_TO_BUFFER_DOMAIN

#define PW_TOKENIZE_TO_BUFFER_DOMAIN	(		domain,
			buffer,
			buffer_size_pointer,
			format,
			...
	)

Value:

  PW_TOKENIZE_TO_BUFFER_MASK(                         \
      domain, UINT32_MAX, buffer, buffer_size_pointer, format, __VA_ARGS__)

Same as PW_TOKENIZE_TO_BUFFER, but tokenizes to the specified domain.

PW_TOKENIZE_TO_BUFFER_MASK

#define PW_TOKENIZE_TO_BUFFER_MASK	(		domain,
			mask,
			buffer,
			buffer_size_pointer,
			format,
			...
	)

Value:

  do {                                                                       \
    PW_TOKENIZE_FORMAT_STRING(domain, mask, format, __VA_ARGS__);            \
    _pw_tokenizer_ToBuffer(buffer,                                           \
                           buffer_size_pointer,                              \
                           PW_TOKENIZER_REPLACE_FORMAT_STRING(__VA_ARGS__)); \
  } while (0)

Same as PW_TOKENIZE_TO_BUFFER_DOMAIN, but applies a bit mask to the token.

PW_TOKENIZER_ARG_TYPES

#define PW_TOKENIZER_ARG_TYPES

(

...

)

PW_DELEGATE_BY_ARG_COUNT(_PW_TOKENIZER_TYPES_, __VA_ARGS__)

Converts a series of arguments to a compact format that replaces the format string literal. Evaluates to a pw_tokenizer_ArgTypes value.

Depending on the size of pw_tokenizer_ArgTypes, the bottom 4 or 6 bits store the number of arguments and the remaining bits store the types, two bits per type. The arguments are not evaluated; only their types are used.

In general, PW_TOKENIZER_ARG_TYPES should not be used directly. Instead, use PW_TOKENIZER_REPLACE_FORMAT_STRING.

PW_TOKENIZER_DEFAULT_DOMAIN

#define PW_TOKENIZER_DEFAULT_DOMAIN   ""

Strings may optionally be tokenized to a domain. Strings in different domains can be processed separately by the token database tools. Each domain in use must have a corresponding section declared in the linker script. See pw_tokenizer_linker_sections.ld for more details.

The default domain is an empty string.

PW_TOKENIZER_DEFINE_TOKEN

#define PW_TOKENIZER_DEFINE_TOKEN	(		token,
			domain,
			string
	)

Value:

  static_assert(::pw::tokenizer::internal::ValidDomain(domain),                \
                "pw_tokenizer domains may only contain alphanumeric "          \
                "characters, underscore, or colon, and cannot start with a "   \
                "number; space characters are ignored");                       \
  alignas(1) static constexpr auto _PW_TOKENIZER_SECTION _PW_TOKENIZER_UNIQUE( \
      _pw_tokenizer_string_entry_) =                                           \
      ::pw::tokenizer::internal::MakeEntry(token, domain, string)

Records the original token, domain and string directly.

This macro is intended to be used for tokenized enum and domain support. The values are stored as an entry in the ELF section. As a note for tokenized enum support, the enum name should be used as the string, and the enum value as the token.

PW_TOKENIZER_REPLACE_FORMAT_STRING

#define PW_TOKENIZER_REPLACE_FORMAT_STRING

(

...

)

_PW_TOKENIZER_REPLACE_FORMAT_STRING(PW_EMPTY_ARGS(__VA_ARGS__), __VA_ARGS__)

Low-level macro for calling functions that handle tokenized strings.

Functions that work with tokenized format strings must take the following arguments:

• The 32-bit token (pw_tokenizer_Token)
• The 32- or 64-bit argument types (pw_tokenizer_ArgTypes)
• Variadic arguments, if any

This macro expands to those arguments. Custom tokenization macros should use this macro to pass these arguments to a function or other macro.

EncodeMyTokenizedString(uint32_t token,
                        pw_tokenier_ArgTypes arg_types,
                        ...);
 
#define CUSTOM_TOKENIZATION_MACRO(format, ...)                  \
  PW_TOKENIZE_FORMAT_STRING(domain, mask, format, __VA_ARGS__); \
  EncodeMyTokenizedString(PW_TOKENIZER_REPLACE_FORMAT_STRING(__VA_ARGS__))

Typedef Documentation

pw_tokenizer_Token

typedef uint32_t pw_tokenizer_Token

The type of the 32-bit token used in place of a string. Also available as pw::tokenizer::Token.

Function Documentation

Base64EncodedBufferSize()

constexpr size_t pw::tokenizer::Base64EncodedBufferSize ( size_t  message_size )

constexpr

Same as Base64EncodedStringSize(), but for sizing char buffers. Includes room for the prefix character ($), encoded message, and a null terminator.

Base64EncodedStringSize()

constexpr size_t pw::tokenizer::Base64EncodedStringSize ( size_t  message_size )

constexpr

Returns the size of a tokenized message (token + arguments) when encoded as prefixed Base64. Includes room for the prefix character ($) and encoded message. This value is the capacity needed to encode to a pw::InlineString.

EncodeArgs()

size_t pw::tokenizer::EncodeArgs	(	pw_tokenizer_ArgTypes	types,
		va_list	args,
		span< std::byte >	output
	)

Encodes a tokenized string's arguments to a buffer. The pw_tokenizer_ArgTypes parameter specifies the argument types, in place of a format string.

Most tokenization implementations should use the pw::tokenizer::EncodedMessage class.

EnumToString()

template<typename T >

constexpr const char * pw::tokenizer::EnumToString ( T  value )

constexpr

Returns a string representation of a given enumerator value name. Used in the case of a non-tokenizing log backend.

Returns a string representation of a given enumerator value name.

Parameters

value

enumerator value

Returns

constexpr char array

EnumToToken()

template<typename T >

constexpr auto pw::tokenizer::EnumToToken ( T  value )

constexpr

Tokenizes a given enumerator value. Used in the vase of a tokenizing log backend.

Parameters

value

enumerator value

Returns

The 32-bit token (pw_tokenizer_Token)

MinEncodingBufferSizeBytes()

template<typename... ArgTypes>

constexpr size_t pw::tokenizer::MinEncodingBufferSizeBytes ( )

constexpr

Calculates the minimum buffer size to allocate that is guaranteed to support encoding the specified arguments.

The contents of strings are NOT included in this total. The string's length/status byte is guaranteed to fit, but the string contents may be truncated. Encoding is considered to succeed as long as the string's length/status byte is written, even if the actual string is truncated.

Examples:

• Message with no arguments: MinEncodingBufferSizeBytes() == 4
• Message with an int argument MinEncodingBufferSizeBytes<int>() == 9 (4 + 5)

PrefixedBase64Decode()

size_t pw::tokenizer::PrefixedBase64Decode ( std::string_view  base64_message, span< std::byte >  output_buffer  )

inline

Decodes a prefixed Base64 tokenized message to binary. Returns the size of the decoded binary data. The resulting data is ready to be passed to pw::tokenizer::Detokenizer::Detokenize.

PrefixedBase64DecodeInPlace() [1/2]

template<typename CharT >

void pw::tokenizer::PrefixedBase64DecodeInPlace ( InlineBasicString< CharT > &  string )

inline

Decodes a prefixed Base64 tokenized message to binary in place. Resizes the string to fit the decoded binary data.

PrefixedBase64DecodeInPlace() [2/2]

size_t pw::tokenizer::PrefixedBase64DecodeInPlace ( span< std::byte >  buffer )

inline

Decodes a prefixed Base64 tokenized message to binary in place. Returns the size of the decoded binary data.

PrefixedBase64Encode() [1/3]

template<size_t kMaxBinaryMessageSizeBytes>

auto pw::tokenizer::PrefixedBase64Encode

(

span< const std::byte >

binary_message

)

Encodes a binary tokenized message as prefixed Base64 to a pw::InlineString. The pw::InlineString is sized to fit messages up to kMaxBinaryMessageSizeBytes long. Asserts if the message is larger.

PrefixedBase64Encode() [2/3]

void pw::tokenizer::PrefixedBase64Encode	(	span< const std::byte >	binary_message,
		InlineString<> &	output
	)

Encodes a binary tokenized message as prefixed Base64 to a pw::InlineString, appending to any existing contents. Asserts if the message does not fit in the string.

PrefixedBase64Encode() [3/3]

size_t pw::tokenizer::PrefixedBase64Encode ( span< const std::byte >  binary_message, span< char >  output_buffer  )

inline

Encodes a binary tokenized message as prefixed Base64 with a null terminator. Returns the encoded string length (excluding the null terminator). Returns 0 if the buffer is too small. Always null terminates if the output buffer is not empty.

pw_tokenizer_EncodeArgs()

size_t pw_tokenizer_EncodeArgs	(	pw_tokenizer_ArgTypes	types,
		va_list	args,
		void *	output_buffer,
		size_t	output_buffer_size
	)

C function that encodes arguments to a tokenized buffer. Use the pw::tokenizer::EncodeArgs function from C++.

pw_tokenizer_EncodeInt()

static size_t pw_tokenizer_EncodeInt ( int  value, void *  output, size_t  output_size_bytes  )

inlinestatic

Encodes an int with the standard integer encoding: zig-zag + LEB128. This function is only necessary when manually encoding tokenized messages.

pw_tokenizer_EncodeInt64()

static size_t pw_tokenizer_EncodeInt64 ( int64_t  value, void *  output, size_t  output_size_bytes  )

inlinestatic

Encodes an int64_t with the standard integer encoding: zig-zag + LEB128. This function is only necessary when manually encoding tokenized messages.

pw_tokenizer_PrefixedBase64Decode()

size_t pw_tokenizer_PrefixedBase64Decode	(	const void *	base64_message,
		size_t	base64_size_bytes,
		void *	output_buffer,
		size_t	output_buffer_size
	)

Decodes a prefixed Base64 tokenized message to binary. Returns the size of the decoded binary data. The resulting data is ready to be passed to pw::tokenizer::Detokenizer::Detokenize. Returns 0 if the buffer is too small, the expected prefix character is missing, or the Base64 data is corrupt.

Equivalent to pw::tokenizer::PrefixedBase64Encode.

pw_tokenizer_PrefixedBase64Encode()

size_t pw_tokenizer_PrefixedBase64Encode	(	const void *	binary_message,
		size_t	binary_size_bytes,
		void *	output_buffer,
		size_t	output_buffer_size_bytes
	)

Encodes a binary tokenized message as prefixed Base64 with a null terminator. Returns the encoded string length (excluding the null terminator). Returns 0 if the buffer is too small. Always null terminates if the output buffer is not empty.

Equivalent to pw::tokenizer::PrefixedBase64Encode.