pw_tokenizer/lib.rs
1// Copyright 2023 The Pigweed Authors
2//
3// Licensed under the Apache License, Version 2.0 (the "License"); you may not
4// use this file except in compliance with the License. You may obtain a copy of
5// the License at
6//
7// https://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12// License for the specific language governing permissions and limitations under
13// the License.
14
15//! `pw_tokenizer` - Efficient string handling and printf style encoding.
16//!
17//! Logging is critical, but developers are often forced to choose between
18//! additional logging or saving crucial flash space. The `pw_tokenizer` crate
19//! helps address this by replacing printf-style strings with binary tokens
20//! during compilation. This enables extensive logging with substantially less
21//! memory usage.
22//!
23//! For a more in depth explanation of the systems design and motivations,
24//! see [Pigweed's pw_tokenizer module documentation](https://pigweed.dev/pw_tokenizer/).
25//!
26//! # Examples
27//!
28//! Pigweed's tokenization database uses `printf` style strings internally so
29//! those are supported directly.
30//!
31//! ```
32//! use pw_tokenizer::tokenize_printf_to_buffer;
33//!
34//! let mut buffer = [0u8; 1024];
35//! let len = tokenize_printf_to_buffer!(&mut buffer, "The answer is %d", 42)?;
36//!
37//! // 4 bytes used to encode the token and one to encode the value 42. This
38//! // is a **3.5x** reduction in size compared to the raw string!
39//! assert_eq!(len, 5);
40//! # Ok::<(), pw_status::Error>(())
41//! ```
42//!
43//! We also support Rust's `core::fmt` style syntax. These format strings are
44//! converted to `printf` style at compile time to maintain compatibly with the
45//! rest of the Pigweed tokenizer ecosystem. The below example produces the
46//! same token and output as the above one.
47//!
48//! ```
49//! use pw_tokenizer::tokenize_core_fmt_to_buffer;
50//!
51//! let mut buffer = [0u8; 1024];
52//! let len = tokenize_core_fmt_to_buffer!(&mut buffer, "The answer is {}", 42 as i32)?;
53//! assert_eq!(len, 5);
54//! # Ok::<(), pw_status::Error>(())
55//! ```
56#![cfg_attr(not(feature = "std"), no_std)]
57#![deny(missing_docs)]
58
59use pw_status::Result;
60
61#[doc(hidden)]
62pub mod internal;
63
64/// Detokenization support.
65#[cfg(feature = "std")]
66pub mod detokenize;
67
68#[doc(hidden)]
69// Creating a __private namespace allows us a way to get to the modules
70// we need from macros by doing:
71// use $crate::__private as __pw_tokenizer_crate;
72//
73// This is how proc macro generated code can reliably reference back to
74// `pw_tokenizer` while still allowing a user to import it under a different
75// name.
76pub mod __private {
77 pub use pw_bytes::concat_static_strs;
78 pub use pw_format_core::{PrintfFormatter, PrintfHexFormatter, PrintfUpperHexFormatter};
79 pub use pw_status::Result;
80 pub use pw_stream::{Cursor, Seek, WriteInteger, WriteVarint};
81 pub use pw_tokenizer_core::hash_string;
82 pub use pw_tokenizer_macro::{
83 _token, _tokenize_core_fmt_to_buffer, _tokenize_core_fmt_to_writer,
84 _tokenize_printf_to_buffer, _tokenize_printf_to_writer,
85 };
86
87 pub use crate::*;
88}
89
90/// Return the [`u32`] token for the specified string and add it to the token
91/// database.
92///
93/// This is where the magic happens in `pw_tokenizer`! ... and by magic
94/// we mean hiding information in a special linker section that ends up in the
95/// final elf binary but does not get flashed to the device.
96///
97/// Two things are accomplished here:
98/// 1) The string is hashed into its stable `u32` token. This is the value that
99/// is returned from the macro.
100/// 2) A [token database entry](https://pigweed.dev/pw_tokenizer/design.html#binary-database-format)
101/// is generated, assigned to a unique static symbol, placed in a linker
102/// section named `pw_tokenizer.entries.<TOKEN_HASH>`. A
103/// [linker script](https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main/pw_tokenizer/pw_tokenizer_linker_sections.ld)
104/// is responsible for picking these symbols up and aggregating them into a
105/// single `.pw_tokenizer.entries` section in the final binary.
106///
107/// # Example
108/// ```
109/// use pw_tokenizer::token;
110///
111/// let token = token!("hello, \"world\"");
112/// assert_eq!(token, 3537412730);
113/// ```
114///
115/// Currently there is no support for encoding tokens to specific domains
116/// or with "fixed lengths" per [`pw_tokenizer_core::hash_bytes_fixed`].
117#[macro_export]
118macro_rules! token {
119 ($string:literal) => {{
120 use $crate::__private as __pw_tokenizer_crate;
121 $crate::__private::_token!($string)
122 }};
123}
124
125/// Tokenize a `core::fmt` style format string and arguments to an [`AsMut<u8>`]
126/// buffer. The format string is converted in to a `printf` and added token to
127/// the token database.
128///
129/// See [`token`] for an explanation on how strings are tokenized and entries
130/// are added to the token database. The token's domain is set to `""`.
131///
132/// Returns a [`pw_status::Result<usize>`] the number of bytes written to the buffer.
133///
134/// `tokenize_to_buffer!` supports concatenation of format strings as described
135/// in [`pw_format::macros::FormatAndArgs`].
136///
137/// # Errors
138/// - [`pw_status::Error::OutOfRange`] - Buffer is not large enough to fit
139/// tokenized data.
140/// - [`pw_status::Error::InvalidArgument`] - Invalid buffer was provided.
141///
142/// # Example
143///
144/// ```
145/// use pw_tokenizer::tokenize_core_fmt_to_buffer;
146///
147/// // Tokenize a format string and argument into a buffer.
148/// let mut buffer = [0u8; 1024];
149/// let len = tokenize_core_fmt_to_buffer!(&mut buffer, "The answer is {}", 42 as i32)?;
150///
151/// // 4 bytes used to encode the token and one to encode the value 42.
152/// assert_eq!(len, 5);
153///
154/// // The format string can be composed of multiple strings literals using
155/// // the custom`PW_FMT_CONCAT` operator.
156/// let len = tokenize_core_fmt_to_buffer!(&mut buffer, "Hello " PW_FMT_CONCAT "Pigweed")?;
157///
158/// // Only a single 4 byte token is emitted after concatenation of the string
159/// // literals above.
160/// assert_eq!(len, 4);
161/// # Ok::<(), pw_status::Error>(())
162/// ```
163#[macro_export]
164macro_rules! tokenize_core_fmt_to_buffer {
165 ($buffer:expr, $($format_string:literal)PW_FMT_CONCAT+ $(, $args:expr)* $(,)?) => {{
166 use $crate::__private as __pw_tokenizer_crate;
167 __pw_tokenizer_crate::_tokenize_core_fmt_to_buffer!($buffer, $($format_string)PW_FMT_CONCAT+, $($args),*)
168 }};
169}
170
171/// Tokenize a printf format string and arguments to an [`AsMut<u8>`] buffer
172/// and add the format string's token to the token database.
173///
174/// See [`token`] for an explanation on how strings are tokenized and entries
175/// are added to the token database. The token's domain is set to `""`.
176///
177/// Returns a [`pw_status::Result<usize>`] the number of bytes written to the buffer.
178///
179/// `tokenize_to_buffer!` supports concatenation of format strings as described
180/// in [`pw_format::macros::FormatAndArgs`].
181///
182/// # Errors
183/// - [`pw_status::Error::OutOfRange`] - Buffer is not large enough to fit
184/// tokenized data.
185/// - [`pw_status::Error::InvalidArgument`] - Invalid buffer was provided.
186///
187/// # Example
188///
189/// ```
190/// use pw_tokenizer::tokenize_printf_to_buffer;
191///
192/// // Tokenize a format string and argument into a buffer.
193/// let mut buffer = [0u8; 1024];
194/// let len = tokenize_printf_to_buffer!(&mut buffer, "The answer is %d", 42)?;
195///
196/// // 4 bytes used to encode the token and one to encode the value 42.
197/// assert_eq!(len, 5);
198///
199/// // The format string can be composed of multiple strings literals using
200/// // the custom`PW_FMT_CONCAT` operator.
201/// let len = tokenize_printf_to_buffer!(&mut buffer, "Hello " PW_FMT_CONCAT "Pigweed")?;
202///
203/// // Only a single 4 byte token is emitted after concatenation of the string
204/// // literals above.
205/// assert_eq!(len, 4);
206/// # Ok::<(), pw_status::Error>(())
207/// ```
208#[macro_export]
209macro_rules! tokenize_printf_to_buffer {
210 ($buffer:expr, $($format_string:literal)PW_FMT_CONCAT+ $(, $args:expr)* $(,)?) => {{
211 use $crate::__private as __pw_tokenizer_crate;
212 __pw_tokenizer_crate::_tokenize_printf_to_buffer!($buffer, $($format_string)PW_FMT_CONCAT+, $($args),*)
213 }};
214}
215
216/// Deprecated alias for [`tokenize_printf_to_buffer!`].
217#[macro_export]
218macro_rules! tokenize_to_buffer {
219 ($buffer:expr, $($format_string:literal)PW_FMT_CONCAT+ $(, $args:expr)* $(,)?) => {{
220 $crate::tokenize_printf_to_buffer!($buffer, $($format_string)PW_FMT_CONCAT+, $($args),*)
221 }};
222}
223
224/// Tokenize a `core::fmt` format string and arguments to a [`MessageWriter`].
225/// The format string is converted in to a `printf` and added token to the token
226/// database.
227///
228/// `tokenize_core_fmt_to_writer!` and the accompanying [`MessageWriter`] trait
229/// provide an optimized API for use cases like logging where the output of the
230/// tokenization will be written to a shared/ambient resource like stdio, a
231/// UART, or a shared buffer.
232///
233/// The `writer_type` should implement [`MessageWriter`] and [`Default`] traits.
234/// The writer is instantiated with the [`Default`] allowing any intermediate
235/// buffers to be declared on the stack of the internal writing engine instead
236/// of the caller's stack.
237///
238/// See [`token`] for an explanation on how strings are tokenized and entries
239/// are added to the token database. The token's domain is set to `""`.
240///
241/// Returns a [`pw_status::Result<()>`].
242///
243/// `tokenize_core_fmt_to_writer!` supports concatenation of format strings as
244/// described in [`pw_format::macros::FormatAndArgs`].
245///
246/// # Errors
247/// - [`pw_status::Error::OutOfRange`] - [`MessageWriter`] does not have enough
248/// space to fit tokenized data.
249/// - others - `tokenize_core_fmt_to_writer!` will pass on any errors returned
250/// by the [`MessageWriter`].
251///
252/// # Code Size
253///
254/// This data was collected by examining the disassembly of a test program
255/// built for a Cortex M0.
256///
257/// | Tokenized Message | Per Call-site Cost (bytes) |
258/// | --------------------| -------------------------- |
259/// | no arguments | 10 |
260/// | one `i32` argument | 18 |
261///
262/// # Example
263///
264/// ```
265/// use pw_status::Result;
266/// use pw_stream::{Cursor, Write};
267/// use pw_tokenizer::{MessageWriter, tokenize_core_fmt_to_writer};
268///
269/// const BUFFER_LEN: usize = 32;
270///
271/// // Declare a simple MessageWriter that uses a [`pw_status::Cursor`] to
272/// // maintain an internal buffer.
273/// struct TestMessageWriter {
274/// cursor: Cursor<[u8; BUFFER_LEN]>,
275/// }
276///
277/// impl Default for TestMessageWriter {
278/// fn default() -> Self {
279/// Self {
280/// cursor: Cursor::new([0u8; BUFFER_LEN]),
281/// }
282/// }
283/// }
284///
285/// impl MessageWriter for TestMessageWriter {
286/// fn write(&mut self, data: &[u8]) -> Result<()> {
287/// self.cursor.write_all(data)
288/// }
289///
290/// fn remaining(&self) -> usize {
291/// self.cursor.remaining()
292/// }
293///
294/// fn finalize(self) -> Result<()> {
295/// let len = self.cursor.position();
296/// // 4 bytes used to encode the token and one to encode the value 42.
297/// assert_eq!(len, 5);
298/// Ok(())
299/// }
300/// }
301///
302/// // Tokenize a format string and argument into the writer. Note how we
303/// // pass in the message writer's type, not an instance of it.
304/// let len = tokenize_core_fmt_to_writer!(TestMessageWriter, "The answer is {}", 42 as i32)?;
305/// # Ok::<(), pw_status::Error>(())
306/// ```
307#[macro_export]
308macro_rules! tokenize_core_fmt_to_writer {
309 ($writer:expr, $($format_string:literal)PW_FMT_CONCAT+ $(, $args:expr)* $(,)?) => {{
310 use $crate::__private as __pw_tokenizer_crate;
311 __pw_tokenizer_crate::_tokenize_core_fmt_to_writer!($writer, $($format_string)PW_FMT_CONCAT+, $($args),*)
312 }};
313}
314
315/// Tokenize a `printf` format string and arguments to a [`MessageWriter`] and
316/// add the format string's token to the token database.
317///
318/// `tokenize_printf_fmt_to_writer!` and the accompanying [`MessageWriter`] trait
319/// provide an optimized API for use cases like logging where the output of the
320/// tokenization will be written to a shared/ambient resource like stdio, a
321/// UART, or a shared buffer.
322///
323/// The `writer_type` should implement [`MessageWriter`] and [`Default`] traits.
324/// The writer is instantiated with the [`Default`] allowing any intermediate
325/// buffers to be declared on the stack of the internal writing engine instead
326/// of the caller's stack.
327///
328/// See [`token`] for an explanation on how strings are tokenized and entries
329/// are added to the token database. The token's domain is set to `""`.
330///
331/// Returns a [`pw_status::Result<()>`].
332///
333/// `tokenize_core_fmt_to_writer!` supports concatenation of format strings as
334/// described in [`pw_format::macros::FormatAndArgs`].
335///
336/// # Errors
337/// - [`pw_status::Error::OutOfRange`] - [`MessageWriter`] does not have enough
338/// space to fit tokenized data.
339/// - others - `tokenize_printf_to_writer!` will pass on any errors returned
340/// by the [`MessageWriter`].
341///
342/// # Code Size
343///
344/// This data was collected by examining the disassembly of a test program
345/// built for a Cortex M0.
346///
347/// | Tokenized Message | Per Call-site Cost (bytes) |
348/// | --------------------| -------------------------- |
349/// | no arguments | 10 |
350/// | one `i32` argument | 18 |
351///
352/// # Example
353///
354/// ```
355/// use pw_status::Result;
356/// use pw_stream::{Cursor, Write};
357/// use pw_tokenizer::{MessageWriter, tokenize_printf_to_writer};
358///
359/// const BUFFER_LEN: usize = 32;
360///
361/// // Declare a simple MessageWriter that uses a [`pw_status::Cursor`] to
362/// // maintain an internal buffer.
363/// struct TestMessageWriter {
364/// cursor: Cursor<[u8; BUFFER_LEN]>,
365/// }
366///
367/// impl Default for TestMessageWriter {
368/// fn default() -> Self {
369/// Self {
370/// cursor: Cursor::new([0u8; BUFFER_LEN]),
371/// }
372/// }
373/// }
374///
375/// impl MessageWriter for TestMessageWriter {
376/// fn write(&mut self, data: &[u8]) -> Result<()> {
377/// self.cursor.write_all(data)
378/// }
379///
380/// fn remaining(&self) -> usize {
381/// self.cursor.remaining()
382/// }
383///
384/// fn finalize(self) -> Result<()> {
385/// let len = self.cursor.position();
386/// // 4 bytes used to encode the token and one to encode the value 42.
387/// assert_eq!(len, 5);
388/// Ok(())
389/// }
390/// }
391///
392/// // Tokenize a format string and argument into the writer. Note how we
393/// // pass in the message writer's type, not an instance of it.
394/// let len = tokenize_printf_to_writer!(TestMessageWriter, "The answer is %d", 42)?;
395/// # Ok::<(), pw_status::Error>(())
396/// ```
397#[macro_export]
398macro_rules! tokenize_printf_to_writer {
399 ($writer:expr, $($format_string:literal)PW_FMT_CONCAT+ $(, $args:expr)* $(,)?) => {{
400 use $crate::__private as __pw_tokenizer_crate;
401 __pw_tokenizer_crate::_tokenize_printf_to_writer!($writer, $($format_string)PW_FMT_CONCAT+, $($args),*)
402 }};
403}
404
405/// Deprecated alias for [`tokenize_printf_to_writer!`].
406#[macro_export]
407macro_rules! tokenize_to_writer {
408 ($writer:expr, $($format_string:literal)PW_FMT_CONCAT+ $(, $args:expr)* $(,)?) => {{
409 $crate::tokenize_printf_to_writer!($writer, $($format_string)PW_FMT_CONCAT+, $($args),*)
410 }};
411}
412
413/// A trait used by [`tokenize_to_writer!`] to output tokenized messages.
414///
415/// For more details on how this type is used, see the [`tokenize_to_writer!`]
416/// documentation.
417pub trait MessageWriter {
418 /// Append `data` to the message.
419 fn write(&mut self, data: &[u8]) -> Result<()>;
420
421 /// Return the remaining space in this message instance.
422 ///
423 /// If there are no space constraints, return `usize::MAX`.
424 fn remaining(&self) -> usize;
425
426 /// Finalize message.
427 ///
428 /// `finalize()` is called when the tokenized message is complete.
429 fn finalize(self) -> Result<()>;
430}
431
432#[cfg(test)]
433// Untyped prints code rely on as casts to annotate type information.
434#[allow(clippy::unnecessary_cast)]
435#[allow(clippy::literal_string_with_formatting_args)]
436mod tests {
437 use super::*;
438 extern crate self as pw_tokenizer;
439 use std::cell::RefCell;
440
441 use pw_stream::{Cursor, Write};
442
443 // This is not meant to be an exhaustive test of tokenization which is
444 // covered by `pw_tokenizer_core`'s unit tests. Rather, this is testing
445 // that the `tokenize!` macro connects to that correctly.
446 #[test]
447 fn test_token() {}
448
449 macro_rules! tokenize_to_buffer_test {
450 ($expected_data:expr, $buffer_len:expr, $printf_fmt:literal, $core_fmt:literal $(, $args:expr)* $(,)?) => {{
451 if $printf_fmt != "" {
452 let mut buffer = [0u8; $buffer_len];
453 let len = tokenize_printf_to_buffer!(&mut buffer, $printf_fmt, $($args),*).unwrap();
454 assert_eq!(
455 &buffer[..len],
456 $expected_data,
457 "printf style input does not produce expected output",
458 );
459 }
460 if $core_fmt != "" {
461 let mut buffer = [0u8; $buffer_len];
462 let len = tokenize_core_fmt_to_buffer!(&mut buffer, $core_fmt, $($args),*).unwrap();
463 assert_eq!(
464 &buffer[..len],
465 $expected_data,
466 "core::fmt style input does not produce expected output",
467 );
468 }
469 }}
470 }
471
472 macro_rules! tokenize_to_writer_test {
473 ($expected_data:expr, $buffer_len:expr, $printf_fmt:literal, $core_fmt:literal $(, $args:expr)* $(,)?) => {{
474 // The `MessageWriter` API is used in places like logging where it
475 // accesses an shared/ambient resource (like stdio or an UART). To test
476 // it in a hermetic way we declare test specific `MessageWriter` that
477 // writes it's output to a scoped static variable that can be checked
478 // after the test is run.
479
480 // Since these tests are not multi-threaded, we can use a thread_local!
481 // instead of a mutex.
482 thread_local!(static TEST_OUTPUT: RefCell<Option<Vec<u8>>> = RefCell::new(None));
483
484 struct TestMessageWriter {
485 cursor: Cursor<[u8; $buffer_len]>,
486 }
487
488 impl Default for TestMessageWriter {
489 fn default() -> Self {
490 Self {
491 cursor: Cursor::new([0u8; $buffer_len]),
492 }
493 }
494 }
495
496 impl MessageWriter for TestMessageWriter {
497 fn write(&mut self, data: &[u8]) -> Result<()> {
498 self.cursor.write_all(data)
499 }
500
501 fn remaining(&self) -> usize {
502 self.cursor.remaining()
503 }
504
505 fn finalize(self) -> Result<()> {
506 let write_len = self.cursor.position();
507 let data = self.cursor.into_inner();
508 TEST_OUTPUT.with(|output| *output.borrow_mut() = Some(data[..write_len].to_vec()));
509
510 Ok(())
511 }
512 }
513
514 if $printf_fmt != "" {
515 TEST_OUTPUT.with(|output| *output.borrow_mut() = None);
516 tokenize_printf_to_writer!(TestMessageWriter, $printf_fmt, $($args),*).unwrap();
517 TEST_OUTPUT.with(|output| {
518 assert_eq!(
519 *output.borrow(),
520 Some($expected_data.to_vec()),
521 )
522 });
523 }
524
525 if $core_fmt != "" {
526 TEST_OUTPUT.with(|output| *output.borrow_mut() = None);
527 tokenize_core_fmt_to_writer!(TestMessageWriter, $core_fmt, $($args),*).unwrap();
528 TEST_OUTPUT.with(|output| {
529 assert_eq!(
530 *output.borrow(),
531 Some($expected_data.to_vec()),
532 )
533 });
534 }
535 }}
536 }
537
538 macro_rules! tokenize_test {
539 ($expected_data:expr, $buffer_len:expr, $printf_fmt:literal, $core_fmt:literal $(, $args:expr)* $(,)?) => {{
540 tokenize_to_buffer_test!($expected_data, $buffer_len, $printf_fmt, $core_fmt, $($args),*);
541 tokenize_to_writer_test!($expected_data, $buffer_len, $printf_fmt, $core_fmt, $($args),*);
542 }};
543 }
544
545 #[test]
546 fn bare_string_encodes_correctly() {
547 tokenize_test!(
548 &[0xe0, 0x92, 0xe0, 0xa], // expected buffer
549 64, // buffer size
550 "Hello Pigweed", // printf style
551 "Hello Pigweed", // core::fmt style
552 );
553 }
554
555 #[test]
556 fn test_decimal_format() {
557 // "as casts" are used for the integer arguments below. They are only
558 // need for the core::fmt style arguments but are added so that we can
559 // check that the printf and core::fmt style equivalents encode the same.
560 tokenize_test!(
561 &[0x52, 0x1c, 0xb0, 0x4c, 0x2], // expected buffer
562 64, // buffer size
563 "The answer is %d!", // printf style
564 "The answer is {}!", // core::fmt style
565 1 as i32
566 );
567
568 tokenize_test!(
569 &[0x36, 0xd0, 0xfb, 0x69, 0x1], // expected buffer
570 64, // buffer size
571 "No! The answer is %d!", // printf style
572 "No! The answer is {}!", // core::fmt style
573 -1 as i32
574 );
575
576 tokenize_test!(
577 &[0xa4, 0xad, 0x50, 0x54, 0x0], // expected buffer
578 64, // buffer size
579 "I think you'll find that the answer is %d!", // printf style
580 "I think you'll find that the answer is {}!", // core::fmt style
581 0 as i32
582 );
583 }
584
585 #[test]
586 fn test_misc_integer_format() {
587 // %d, %i, %o, %u, %x, %X all encode integers the same.
588 tokenize_test!(
589 &[0x52, 0x1c, 0xb0, 0x4c, 0x2], // expected buffer
590 64, // buffer size
591 "The answer is %d!", // printf style
592 "", // no equivalent core::fmt style
593 1
594 );
595
596 // Because %i is an alias for %d, it gets converted to a %d by the
597 // `pw_format` macro infrastructure.
598 tokenize_test!(
599 &[0x52, 0x1c, 0xb0, 0x4c, 0x2], // expected buffer
600 64, // buffer size
601 "The answer is %i!", // printf style
602 "", // no equivalent core::fmt style
603 1
604 );
605
606 tokenize_test!(
607 &[0x5d, 0x70, 0x12, 0xb4, 0x2], // expected buffer
608 64, // buffer size
609 "The answer is %o!", // printf style
610 "", // no equivalent core::fmt style
611 1u32
612 );
613
614 tokenize_test!(
615 &[0x63, 0x58, 0x5f, 0x8f, 0x2], // expected buffer
616 64, // buffer size
617 "The answer is %u!", // printf style
618 "", // no equivalent core::fmt style
619 1u32
620 );
621
622 tokenize_test!(
623 &[0x66, 0xcc, 0x05, 0x7d, 0x2], // expected buffer
624 64, // buffer size
625 "The answer is %x!", // printf style
626 "", // no equivalent core::fmt style
627 1u32
628 );
629
630 tokenize_test!(
631 &[0x46, 0x4c, 0x16, 0x96, 0x2], // expected buffer
632 64, // buffer size
633 "The answer is %X!", // printf style
634 "", // no equivalent core::fmt style
635 1u32
636 );
637 }
638
639 #[test]
640 fn test_string_format() {
641 tokenize_test!(
642 b"\x25\xf6\x2e\x66\x07Pigweed", // expected buffer
643 64, // buffer size
644 "Hello: %s!", // printf style
645 "", // no equivalent core::fmt style
646 "Pigweed"
647 );
648 }
649
650 #[test]
651 fn test_string_format_overflow() {
652 tokenize_test!(
653 b"\x25\xf6\x2e\x66\x83Pig", // expected buffer
654 8, // buffer size
655 "Hello: %s!", // printf style
656 "", // no equivalent core::fmt style
657 "Pigweed"
658 );
659 }
660
661 #[test]
662 fn test_char_format() {
663 tokenize_test!(
664 &[0x2e, 0x52, 0xac, 0xe4, 0xa0, 0x1], // expected buffer
665 64, // buffer size
666 "Hello: %cigweed", // printf style
667 "", // no equivalent core::fmt style
668 "P".as_bytes()[0]
669 );
670 }
671
672 #[test]
673 fn test_untyped_format() {
674 tokenize_test!(
675 &[0x63, 0x58, 0x5f, 0x8f, 0x2], // expected buffer
676 64, // buffer size
677 "The answer is %u!", // printf style
678 "The answer is {}!", // core::fmt style
679 1 as u32
680 );
681
682 tokenize_test!(
683 &[0x36, 0xd0, 0xfb, 0x69, 0x1], // expected buffer
684 64, // buffer size
685 "No! The answer is %v!", // printf style
686 "No! The answer is {}!", // core::fmt style
687 -1 as i32
688 );
689
690 tokenize_test!(
691 b"\x25\xf6\x2e\x66\x07Pigweed", // expected buffer
692 64, // buffer size
693 "Hello: %v!", // printf style
694 "Hello: {}!", // core::fmt style
695 "Pigweed" as &str
696 );
697 }
698
699 #[test]
700 fn test_field_width_and_zero_pad_format() {
701 tokenize_test!(
702 &[0x3a, 0xc2, 0x1a, 0x05, 0xfc, 0xab, 0x06], // expected buffer
703 64, // buffer size
704 "Lets go to the %x", // printf style
705 "Lets go to the {:x}", // core::fmt style
706 0xcafe as u32
707 );
708
709 tokenize_test!(
710 &[0xf3, 0x16, 0x03, 0x99, 0xfc, 0xab, 0x06], // expected buffer
711 64, // buffer size
712 "Lets go to the %8x", // printf style
713 "Lets go to the {:8x}", // core::fmt style
714 0xcafe as u32
715 );
716
717 tokenize_test!(
718 &[0x44, 0xce, 0xa3, 0x7e, 0xfc, 0xab, 0x06], // expected buffer
719 64, // buffer size
720 "Lets go to the %08x", // printf style
721 "Lets go to the {:08x}", // core::fmt style
722 0xcafe as u32
723 );
724 }
725
726 #[test]
727 fn tokenizer_supports_concatenated_printf_format_strings() {
728 // Since the no argument and some arguments cases are handled differently
729 // by `tokenize_to_buffer!` we need to test both.
730 let mut buffer = [0u8; 64];
731 let len =
732 tokenize_printf_to_buffer!(&mut buffer, "Hello" PW_FMT_CONCAT " Pigweed").unwrap();
733 assert_eq!(&buffer[..len], &[0xe0, 0x92, 0xe0, 0xa]);
734
735 let len = tokenize_printf_to_buffer!(&mut buffer, "Hello: " PW_FMT_CONCAT "%cigweed",
736 "P".as_bytes()[0])
737 .unwrap();
738 assert_eq!(&buffer[..len], &[0x2e, 0x52, 0xac, 0xe4, 0xa0, 0x1]);
739 }
740
741 #[test]
742 fn tokenizer_supports_concatenated_core_fmt_format_strings() {
743 // Since the no argument and some arguments cases are handled differently
744 // by `tokenize_to_buffer!` we need to test both.
745 let mut buffer = [0u8; 64];
746 let len =
747 tokenize_core_fmt_to_buffer!(&mut buffer, "Hello" PW_FMT_CONCAT " Pigweed").unwrap();
748 assert_eq!(&buffer[..len], &[0xe0, 0x92, 0xe0, 0xa]);
749
750 let len = tokenize_core_fmt_to_buffer!(&mut buffer, "The answer is " PW_FMT_CONCAT "{}!",
751 1 as i32)
752 .unwrap();
753 assert_eq!(&buffer[..len], &[0x52, 0x1c, 0xb0, 0x4c, 0x2]);
754 }
755}