pw_tokenizer/lib.rs
1// Copyright 2023 The Pigweed Authors
2//
3// Licensed under the Apache License, Version 2.0 (the "License"); you may not
4// use this file except in compliance with the License. You may obtain a copy of
5// the License at
6//
7// https://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12// License for the specific language governing permissions and limitations under
13// the License.
14
15//! `pw_tokenizer` - Efficient string handling and printf style encoding.
16//!
17//! Logging is critical, but developers are often forced to choose between
18//! additional logging or saving crucial flash space. The `pw_tokenizer` crate
19//! helps address this by replacing printf-style strings with binary tokens
20//! during compilation. This enables extensive logging with substantially less
21//! memory usage.
22//!
23//! For a more in depth explanation of the systems design and motivations,
24//! see [Pigweed's pw_tokenizer module documentation](https://pigweed.dev/pw_tokenizer/).
25//!
26//! # Examples
27//!
28//! Pigweed's tokenization database uses `printf` style strings internally so
29//! those are supported directly.
30//!
31//! ```
32//! use pw_tokenizer::tokenize_printf_to_buffer;
33//!
34//! let mut buffer = [0u8; 1024];
35//! let len = tokenize_printf_to_buffer!(&mut buffer, "The answer is %d", 42)?;
36//!
37//! // 4 bytes used to encode the token and one to encode the value 42. This
38//! // is a **3.5x** reduction in size compared to the raw string!
39//! assert_eq!(len, 5);
40//! # Ok::<(), pw_status::Error>(())
41//! ```
42//!
43//! We also support Rust's `core::fmt` style syntax. These format strings are
44//! converted to `printf` style at compile time to maintain compatibly with the
45//! rest of the Pigweed tokenizer ecosystem. The below example produces the
46//! same token and output as the above one.
47//!
48//! ```
49//! use pw_tokenizer::tokenize_core_fmt_to_buffer;
50//!
51//! let mut buffer = [0u8; 1024];
52//! let len = tokenize_core_fmt_to_buffer!(&mut buffer, "The answer is {}", 42 as i32)?;
53//! assert_eq!(len, 5);
54//! # Ok::<(), pw_status::Error>(())
55//! ```
56#![cfg_attr(feature = "nightly", feature(type_alias_impl_trait))]
57#![cfg_attr(not(feature = "std"), no_std)]
58#![deny(missing_docs)]
59
60use pw_status::Result;
61
62#[doc(hidden)]
63pub mod internal;
64
65#[doc(hidden)]
66// Creating a __private namespace allows us a way to get to the modules
67// we need from macros by doing:
68// use $crate::__private as __pw_tokenizer_crate;
69//
70// This is how proc macro generated code can reliably reference back to
71// `pw_tokenizer` while still allowing a user to import it under a different
72// name.
73pub mod __private {
74 pub use pw_bytes::concat_static_strs;
75 pub use pw_format_core::{PrintfFormatter, PrintfHexFormatter, PrintfUpperHexFormatter};
76 pub use pw_status::Result;
77 pub use pw_stream::{Cursor, Seek, WriteInteger, WriteVarint};
78 pub use pw_tokenizer_core::hash_string;
79 pub use pw_tokenizer_macro::{
80 _token, _tokenize_core_fmt_to_buffer, _tokenize_core_fmt_to_writer,
81 _tokenize_printf_to_buffer, _tokenize_printf_to_writer,
82 };
83
84 pub use crate::*;
85}
86
87/// Return the [`u32`] token for the specified string and add it to the token
88/// database.
89///
90/// This is where the magic happens in `pw_tokenizer`! ... and by magic
91/// we mean hiding information in a special linker section that ends up in the
92/// final elf binary but does not get flashed to the device.
93///
94/// Two things are accomplished here:
95/// 1) The string is hashed into its stable `u32` token. This is the value that
96/// is returned from the macro.
97/// 2) A [token database entry](https://pigweed.dev/pw_tokenizer/design.html#binary-database-format)
98/// is generated, assigned to a unique static symbol, placed in a linker
99/// section named `pw_tokenizer.entries.<TOKEN_HASH>`. A
100/// [linker script](https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main/pw_tokenizer/pw_tokenizer_linker_sections.ld)
101/// is responsible for picking these symbols up and aggregating them into a
102/// single `.pw_tokenizer.entries` section in the final binary.
103///
104/// # Example
105/// ```
106/// use pw_tokenizer::token;
107///
108/// let token = token!("hello, \"world\"");
109/// assert_eq!(token, 3537412730);
110/// ```
111///
112/// Currently there is no support for encoding tokens to specific domains
113/// or with "fixed lengths" per [`pw_tokenizer_core::hash_bytes_fixed`].
114#[macro_export]
115macro_rules! token {
116 ($string:literal) => {{
117 use $crate::__private as __pw_tokenizer_crate;
118 $crate::__private::_token!($string)
119 }};
120}
121
122/// Tokenize a `core::fmt` style format string and arguments to an [`AsMut<u8>`]
123/// buffer. The format string is converted in to a `printf` and added token to
124/// the token database.
125///
126/// See [`token`] for an explanation on how strings are tokenized and entries
127/// are added to the token database. The token's domain is set to `""`.
128///
129/// Returns a [`pw_status::Result<usize>`] the number of bytes written to the buffer.
130///
131/// `tokenize_to_buffer!` supports concatenation of format strings as described
132/// in [`pw_format::macros::FormatAndArgs`].
133///
134/// # Errors
135/// - [`pw_status::Error::OutOfRange`] - Buffer is not large enough to fit
136/// tokenized data.
137/// - [`pw_status::Error::InvalidArgument`] - Invalid buffer was provided.
138///
139/// # Example
140///
141/// ```
142/// use pw_tokenizer::tokenize_core_fmt_to_buffer;
143///
144/// // Tokenize a format string and argument into a buffer.
145/// let mut buffer = [0u8; 1024];
146/// let len = tokenize_core_fmt_to_buffer!(&mut buffer, "The answer is {}", 42 as i32)?;
147///
148/// // 4 bytes used to encode the token and one to encode the value 42.
149/// assert_eq!(len, 5);
150///
151/// // The format string can be composed of multiple strings literals using
152/// // the custom`PW_FMT_CONCAT` operator.
153/// let len = tokenize_core_fmt_to_buffer!(&mut buffer, "Hello " PW_FMT_CONCAT "Pigweed")?;
154///
155/// // Only a single 4 byte token is emitted after concatenation of the string
156/// // literals above.
157/// assert_eq!(len, 4);
158/// # Ok::<(), pw_status::Error>(())
159/// ```
160#[macro_export]
161macro_rules! tokenize_core_fmt_to_buffer {
162 ($buffer:expr, $($format_string:literal)PW_FMT_CONCAT+ $(, $args:expr)* $(,)?) => {{
163 use $crate::__private as __pw_tokenizer_crate;
164 __pw_tokenizer_crate::_tokenize_core_fmt_to_buffer!($buffer, $($format_string)PW_FMT_CONCAT+, $($args),*)
165 }};
166}
167
168/// Tokenize a printf format string and arguments to an [`AsMut<u8>`] buffer
169/// and add the format string's token to the token database.
170///
171/// See [`token`] for an explanation on how strings are tokenized and entries
172/// are added to the token database. The token's domain is set to `""`.
173///
174/// Returns a [`pw_status::Result<usize>`] the number of bytes written to the buffer.
175///
176/// `tokenize_to_buffer!` supports concatenation of format strings as described
177/// in [`pw_format::macros::FormatAndArgs`].
178///
179/// # Errors
180/// - [`pw_status::Error::OutOfRange`] - Buffer is not large enough to fit
181/// tokenized data.
182/// - [`pw_status::Error::InvalidArgument`] - Invalid buffer was provided.
183///
184/// # Example
185///
186/// ```
187/// use pw_tokenizer::tokenize_printf_to_buffer;
188///
189/// // Tokenize a format string and argument into a buffer.
190/// let mut buffer = [0u8; 1024];
191/// let len = tokenize_printf_to_buffer!(&mut buffer, "The answer is %d", 42)?;
192///
193/// // 4 bytes used to encode the token and one to encode the value 42.
194/// assert_eq!(len, 5);
195///
196/// // The format string can be composed of multiple strings literals using
197/// // the custom`PW_FMT_CONCAT` operator.
198/// let len = tokenize_printf_to_buffer!(&mut buffer, "Hello " PW_FMT_CONCAT "Pigweed")?;
199///
200/// // Only a single 4 byte token is emitted after concatenation of the string
201/// // literals above.
202/// assert_eq!(len, 4);
203/// # Ok::<(), pw_status::Error>(())
204/// ```
205#[macro_export]
206macro_rules! tokenize_printf_to_buffer {
207 ($buffer:expr, $($format_string:literal)PW_FMT_CONCAT+ $(, $args:expr)* $(,)?) => {{
208 use $crate::__private as __pw_tokenizer_crate;
209 __pw_tokenizer_crate::_tokenize_printf_to_buffer!($buffer, $($format_string)PW_FMT_CONCAT+, $($args),*)
210 }};
211}
212
213/// Deprecated alias for [`tokenize_printf_to_buffer!`].
214#[macro_export]
215macro_rules! tokenize_to_buffer {
216 ($buffer:expr, $($format_string:literal)PW_FMT_CONCAT+ $(, $args:expr)* $(,)?) => {{
217 $crate::tokenize_printf_to_buffer!($buffer, $($format_string)PW_FMT_CONCAT+, $($args),*)
218 }};
219}
220
221/// Tokenize a `core::fmt` format string and arguments to a [`MessageWriter`].
222/// The format string is converted in to a `printf` and added token to the token
223/// database.
224///
225/// `tokenize_core_fmt_to_writer!` and the accompanying [`MessageWriter`] trait
226/// provide an optimized API for use cases like logging where the output of the
227/// tokenization will be written to a shared/ambient resource like stdio, a
228/// UART, or a shared buffer.
229///
230/// The `writer_type` should implement [`MessageWriter`] and [`Default`] traits.
231/// The writer is instantiated with the [`Default`] allowing any intermediate
232/// buffers to be declared on the stack of the internal writing engine instead
233/// of the caller's stack.
234///
235/// See [`token`] for an explanation on how strings are tokenized and entries
236/// are added to the token database. The token's domain is set to `""`.
237///
238/// Returns a [`pw_status::Result<()>`].
239///
240/// `tokenize_core_fmt_to_writer!` supports concatenation of format strings as
241/// described in [`pw_format::macros::FormatAndArgs`].
242///
243/// # Errors
244/// - [`pw_status::Error::OutOfRange`] - [`MessageWriter`] does not have enough
245/// space to fit tokenized data.
246/// - others - `tokenize_core_fmt_to_writer!` will pass on any errors returned
247/// by the [`MessageWriter`].
248///
249/// # Code Size
250///
251/// This data was collected by examining the disassembly of a test program
252/// built for a Cortex M0.
253///
254/// | Tokenized Message | Per Call-site Cost (bytes) |
255/// | --------------------| -------------------------- |
256/// | no arguments | 10 |
257/// | one `i32` argument | 18 |
258///
259/// # Example
260///
261/// ```
262/// use pw_status::Result;
263/// use pw_stream::{Cursor, Write};
264/// use pw_tokenizer::{MessageWriter, tokenize_core_fmt_to_writer};
265///
266/// const BUFFER_LEN: usize = 32;
267///
268/// // Declare a simple MessageWriter that uses a [`pw_status::Cursor`] to
269/// // maintain an internal buffer.
270/// struct TestMessageWriter {
271/// cursor: Cursor<[u8; BUFFER_LEN]>,
272/// }
273///
274/// impl Default for TestMessageWriter {
275/// fn default() -> Self {
276/// Self {
277/// cursor: Cursor::new([0u8; BUFFER_LEN]),
278/// }
279/// }
280/// }
281///
282/// impl MessageWriter for TestMessageWriter {
283/// fn write(&mut self, data: &[u8]) -> Result<()> {
284/// self.cursor.write_all(data)
285/// }
286///
287/// fn remaining(&self) -> usize {
288/// self.cursor.remaining()
289/// }
290///
291/// fn finalize(self) -> Result<()> {
292/// let len = self.cursor.position();
293/// // 4 bytes used to encode the token and one to encode the value 42.
294/// assert_eq!(len, 5);
295/// Ok(())
296/// }
297/// }
298///
299/// // Tokenize a format string and argument into the writer. Note how we
300/// // pass in the message writer's type, not an instance of it.
301/// let len = tokenize_core_fmt_to_writer!(TestMessageWriter, "The answer is {}", 42 as i32)?;
302/// # Ok::<(), pw_status::Error>(())
303/// ```
304#[macro_export]
305macro_rules! tokenize_core_fmt_to_writer {
306 ($writer:expr, $($format_string:literal)PW_FMT_CONCAT+ $(, $args:expr)* $(,)?) => {{
307 use $crate::__private as __pw_tokenizer_crate;
308 __pw_tokenizer_crate::_tokenize_core_fmt_to_writer!($writer, $($format_string)PW_FMT_CONCAT+, $($args),*)
309 }};
310}
311
312/// Tokenize a `printf` format string and arguments to a [`MessageWriter`] and
313/// add the format string's token to the token database.
314///
315/// `tokenize_printf_fmt_to_writer!` and the accompanying [`MessageWriter`] trait
316/// provide an optimized API for use cases like logging where the output of the
317/// tokenization will be written to a shared/ambient resource like stdio, a
318/// UART, or a shared buffer.
319///
320/// The `writer_type` should implement [`MessageWriter`] and [`Default`] traits.
321/// The writer is instantiated with the [`Default`] allowing any intermediate
322/// buffers to be declared on the stack of the internal writing engine instead
323/// of the caller's stack.
324///
325/// See [`token`] for an explanation on how strings are tokenized and entries
326/// are added to the token database. The token's domain is set to `""`.
327///
328/// Returns a [`pw_status::Result<()>`].
329///
330/// `tokenize_core_fmt_to_writer!` supports concatenation of format strings as
331/// described in [`pw_format::macros::FormatAndArgs`].
332///
333/// # Errors
334/// - [`pw_status::Error::OutOfRange`] - [`MessageWriter`] does not have enough
335/// space to fit tokenized data.
336/// - others - `tokenize_printf_to_writer!` will pass on any errors returned
337/// by the [`MessageWriter`].
338///
339/// # Code Size
340///
341/// This data was collected by examining the disassembly of a test program
342/// built for a Cortex M0.
343///
344/// | Tokenized Message | Per Call-site Cost (bytes) |
345/// | --------------------| -------------------------- |
346/// | no arguments | 10 |
347/// | one `i32` argument | 18 |
348///
349/// # Example
350///
351/// ```
352/// use pw_status::Result;
353/// use pw_stream::{Cursor, Write};
354/// use pw_tokenizer::{MessageWriter, tokenize_printf_to_writer};
355///
356/// const BUFFER_LEN: usize = 32;
357///
358/// // Declare a simple MessageWriter that uses a [`pw_status::Cursor`] to
359/// // maintain an internal buffer.
360/// struct TestMessageWriter {
361/// cursor: Cursor<[u8; BUFFER_LEN]>,
362/// }
363///
364/// impl Default for TestMessageWriter {
365/// fn default() -> Self {
366/// Self {
367/// cursor: Cursor::new([0u8; BUFFER_LEN]),
368/// }
369/// }
370/// }
371///
372/// impl MessageWriter for TestMessageWriter {
373/// fn write(&mut self, data: &[u8]) -> Result<()> {
374/// self.cursor.write_all(data)
375/// }
376///
377/// fn remaining(&self) -> usize {
378/// self.cursor.remaining()
379/// }
380///
381/// fn finalize(self) -> Result<()> {
382/// let len = self.cursor.position();
383/// // 4 bytes used to encode the token and one to encode the value 42.
384/// assert_eq!(len, 5);
385/// Ok(())
386/// }
387/// }
388///
389/// // Tokenize a format string and argument into the writer. Note how we
390/// // pass in the message writer's type, not an instance of it.
391/// let len = tokenize_printf_to_writer!(TestMessageWriter, "The answer is %d", 42)?;
392/// # Ok::<(), pw_status::Error>(())
393/// ```
394#[macro_export]
395macro_rules! tokenize_printf_to_writer {
396 ($writer:expr, $($format_string:literal)PW_FMT_CONCAT+ $(, $args:expr)* $(,)?) => {{
397 use $crate::__private as __pw_tokenizer_crate;
398 __pw_tokenizer_crate::_tokenize_printf_to_writer!($writer, $($format_string)PW_FMT_CONCAT+, $($args),*)
399 }};
400}
401
402/// Deprecated alias for [`tokenize_printf_to_writer!`].
403#[macro_export]
404macro_rules! tokenize_to_writer {
405 ($writer:expr, $($format_string:literal)PW_FMT_CONCAT+ $(, $args:expr)* $(,)?) => {{
406 $crate::tokenize_printf_to_writer!($writer, $($format_string)PW_FMT_CONCAT+, $($args),*)
407 }};
408}
409
410/// A trait used by [`tokenize_to_writer!`] to output tokenized messages.
411///
412/// For more details on how this type is used, see the [`tokenize_to_writer!`]
413/// documentation.
414pub trait MessageWriter {
415 /// Append `data` to the message.
416 fn write(&mut self, data: &[u8]) -> Result<()>;
417
418 /// Return the remaining space in this message instance.
419 ///
420 /// If there are no space constraints, return `usize::MAX`.
421 fn remaining(&self) -> usize;
422
423 /// Finalize message.
424 ///
425 /// `finalize()` is called when the tokenized message is complete.
426 fn finalize(self) -> Result<()>;
427}
428
429#[cfg(test)]
430// Untyped prints code rely on as casts to annotate type information.
431#[allow(clippy::unnecessary_cast)]
432#[allow(clippy::literal_string_with_formatting_args)]
433mod tests {
434 use super::*;
435 extern crate self as pw_tokenizer;
436 use std::cell::RefCell;
437
438 use pw_stream::{Cursor, Write};
439
440 // This is not meant to be an exhaustive test of tokenization which is
441 // covered by `pw_tokenizer_core`'s unit tests. Rather, this is testing
442 // that the `tokenize!` macro connects to that correctly.
443 #[test]
444 fn test_token() {}
445
446 macro_rules! tokenize_to_buffer_test {
447 ($expected_data:expr, $buffer_len:expr, $printf_fmt:literal, $core_fmt:literal $(, $args:expr)* $(,)?) => {{
448 if $printf_fmt != "" {
449 let mut buffer = [0u8; $buffer_len];
450 let len = tokenize_printf_to_buffer!(&mut buffer, $printf_fmt, $($args),*).unwrap();
451 assert_eq!(
452 &buffer[..len],
453 $expected_data,
454 "printf style input does not produce expected output",
455 );
456 }
457 if $core_fmt != "" {
458 let mut buffer = [0u8; $buffer_len];
459 let len = tokenize_core_fmt_to_buffer!(&mut buffer, $core_fmt, $($args),*).unwrap();
460 assert_eq!(
461 &buffer[..len],
462 $expected_data,
463 "core::fmt style input does not produce expected output",
464 );
465 }
466 }}
467 }
468
469 macro_rules! tokenize_to_writer_test {
470 ($expected_data:expr, $buffer_len:expr, $printf_fmt:literal, $core_fmt:literal $(, $args:expr)* $(,)?) => {{
471 // The `MessageWriter` API is used in places like logging where it
472 // accesses an shared/ambient resource (like stdio or an UART). To test
473 // it in a hermetic way we declare test specific `MessageWriter` that
474 // writes it's output to a scoped static variable that can be checked
475 // after the test is run.
476
477 // Since these tests are not multi-threaded, we can use a thread_local!
478 // instead of a mutex.
479 thread_local!(static TEST_OUTPUT: RefCell<Option<Vec<u8>>> = RefCell::new(None));
480
481 struct TestMessageWriter {
482 cursor: Cursor<[u8; $buffer_len]>,
483 }
484
485 impl Default for TestMessageWriter {
486 fn default() -> Self {
487 Self {
488 cursor: Cursor::new([0u8; $buffer_len]),
489 }
490 }
491 }
492
493 impl MessageWriter for TestMessageWriter {
494 fn write(&mut self, data: &[u8]) -> Result<()> {
495 self.cursor.write_all(data)
496 }
497
498 fn remaining(&self) -> usize {
499 self.cursor.remaining()
500 }
501
502 fn finalize(self) -> Result<()> {
503 let write_len = self.cursor.position();
504 let data = self.cursor.into_inner();
505 TEST_OUTPUT.with(|output| *output.borrow_mut() = Some(data[..write_len].to_vec()));
506
507 Ok(())
508 }
509 }
510
511 if $printf_fmt != "" {
512 TEST_OUTPUT.with(|output| *output.borrow_mut() = None);
513 tokenize_printf_to_writer!(TestMessageWriter, $printf_fmt, $($args),*).unwrap();
514 TEST_OUTPUT.with(|output| {
515 assert_eq!(
516 *output.borrow(),
517 Some($expected_data.to_vec()),
518 )
519 });
520 }
521
522 if $core_fmt != "" {
523 TEST_OUTPUT.with(|output| *output.borrow_mut() = None);
524 tokenize_core_fmt_to_writer!(TestMessageWriter, $core_fmt, $($args),*).unwrap();
525 TEST_OUTPUT.with(|output| {
526 assert_eq!(
527 *output.borrow(),
528 Some($expected_data.to_vec()),
529 )
530 });
531 }
532 }}
533 }
534
535 macro_rules! tokenize_test {
536 ($expected_data:expr, $buffer_len:expr, $printf_fmt:literal, $core_fmt:literal $(, $args:expr)* $(,)?) => {{
537 tokenize_to_buffer_test!($expected_data, $buffer_len, $printf_fmt, $core_fmt, $($args),*);
538 tokenize_to_writer_test!($expected_data, $buffer_len, $printf_fmt, $core_fmt, $($args),*);
539 }};
540 }
541
542 #[test]
543 fn bare_string_encodes_correctly() {
544 tokenize_test!(
545 &[0xe0, 0x92, 0xe0, 0xa], // expected buffer
546 64, // buffer size
547 "Hello Pigweed", // printf style
548 "Hello Pigweed", // core::fmt style
549 );
550 }
551
552 #[test]
553 fn test_decimal_format() {
554 // "as casts" are used for the integer arguments below. They are only
555 // need for the core::fmt style arguments but are added so that we can
556 // check that the printf and core::fmt style equivalents encode the same.
557 tokenize_test!(
558 &[0x52, 0x1c, 0xb0, 0x4c, 0x2], // expected buffer
559 64, // buffer size
560 "The answer is %d!", // printf style
561 "The answer is {}!", // core::fmt style
562 1 as i32
563 );
564
565 tokenize_test!(
566 &[0x36, 0xd0, 0xfb, 0x69, 0x1], // expected buffer
567 64, // buffer size
568 "No! The answer is %d!", // printf style
569 "No! The answer is {}!", // core::fmt style
570 -1 as i32
571 );
572
573 tokenize_test!(
574 &[0xa4, 0xad, 0x50, 0x54, 0x0], // expected buffer
575 64, // buffer size
576 "I think you'll find that the answer is %d!", // printf style
577 "I think you'll find that the answer is {}!", // core::fmt style
578 0 as i32
579 );
580 }
581
582 #[test]
583 fn test_misc_integer_format() {
584 // %d, %i, %o, %u, %x, %X all encode integers the same.
585 tokenize_test!(
586 &[0x52, 0x1c, 0xb0, 0x4c, 0x2], // expected buffer
587 64, // buffer size
588 "The answer is %d!", // printf style
589 "", // no equivalent core::fmt style
590 1
591 );
592
593 // Because %i is an alias for %d, it gets converted to a %d by the
594 // `pw_format` macro infrastructure.
595 tokenize_test!(
596 &[0x52, 0x1c, 0xb0, 0x4c, 0x2], // expected buffer
597 64, // buffer size
598 "The answer is %i!", // printf style
599 "", // no equivalent core::fmt style
600 1
601 );
602
603 tokenize_test!(
604 &[0x5d, 0x70, 0x12, 0xb4, 0x2], // expected buffer
605 64, // buffer size
606 "The answer is %o!", // printf style
607 "", // no equivalent core::fmt style
608 1u32
609 );
610
611 tokenize_test!(
612 &[0x63, 0x58, 0x5f, 0x8f, 0x2], // expected buffer
613 64, // buffer size
614 "The answer is %u!", // printf style
615 "", // no equivalent core::fmt style
616 1u32
617 );
618
619 tokenize_test!(
620 &[0x66, 0xcc, 0x05, 0x7d, 0x2], // expected buffer
621 64, // buffer size
622 "The answer is %x!", // printf style
623 "", // no equivalent core::fmt style
624 1u32
625 );
626
627 tokenize_test!(
628 &[0x46, 0x4c, 0x16, 0x96, 0x2], // expected buffer
629 64, // buffer size
630 "The answer is %X!", // printf style
631 "", // no equivalent core::fmt style
632 1u32
633 );
634 }
635
636 #[test]
637 fn test_string_format() {
638 tokenize_test!(
639 b"\x25\xf6\x2e\x66\x07Pigweed", // expected buffer
640 64, // buffer size
641 "Hello: %s!", // printf style
642 "", // no equivalent core::fmt style
643 "Pigweed"
644 );
645 }
646
647 #[test]
648 fn test_string_format_overflow() {
649 tokenize_test!(
650 b"\x25\xf6\x2e\x66\x83Pig", // expected buffer
651 8, // buffer size
652 "Hello: %s!", // printf style
653 "", // no equivalent core::fmt style
654 "Pigweed"
655 );
656 }
657
658 #[test]
659 fn test_char_format() {
660 tokenize_test!(
661 &[0x2e, 0x52, 0xac, 0xe4, 0xa0, 0x1], // expected buffer
662 64, // buffer size
663 "Hello: %cigweed", // printf style
664 "", // no equivalent core::fmt style
665 "P".as_bytes()[0]
666 );
667 }
668
669 #[test]
670 fn test_untyped_format() {
671 tokenize_test!(
672 &[0x63, 0x58, 0x5f, 0x8f, 0x2], // expected buffer
673 64, // buffer size
674 "The answer is %u!", // printf style
675 "The answer is {}!", // core::fmt style
676 1 as u32
677 );
678
679 tokenize_test!(
680 &[0x36, 0xd0, 0xfb, 0x69, 0x1], // expected buffer
681 64, // buffer size
682 "No! The answer is %v!", // printf style
683 "No! The answer is {}!", // core::fmt style
684 -1 as i32
685 );
686
687 tokenize_test!(
688 b"\x25\xf6\x2e\x66\x07Pigweed", // expected buffer
689 64, // buffer size
690 "Hello: %v!", // printf style
691 "Hello: {}!", // core::fmt style
692 "Pigweed" as &str
693 );
694 }
695
696 #[test]
697 fn test_field_width_and_zero_pad_format() {
698 tokenize_test!(
699 &[0x3a, 0xc2, 0x1a, 0x05, 0xfc, 0xab, 0x06], // expected buffer
700 64, // buffer size
701 "Lets go to the %x", // printf style
702 "Lets go to the {:x}", // core::fmt style
703 0xcafe as u32
704 );
705
706 tokenize_test!(
707 &[0xf3, 0x16, 0x03, 0x99, 0xfc, 0xab, 0x06], // expected buffer
708 64, // buffer size
709 "Lets go to the %8x", // printf style
710 "Lets go to the {:8x}", // core::fmt style
711 0xcafe as u32
712 );
713
714 tokenize_test!(
715 &[0x44, 0xce, 0xa3, 0x7e, 0xfc, 0xab, 0x06], // expected buffer
716 64, // buffer size
717 "Lets go to the %08x", // printf style
718 "Lets go to the {:08x}", // core::fmt style
719 0xcafe as u32
720 );
721 }
722
723 #[test]
724 fn tokenizer_supports_concatenated_printf_format_strings() {
725 // Since the no argument and some arguments cases are handled differently
726 // by `tokenize_to_buffer!` we need to test both.
727 let mut buffer = [0u8; 64];
728 let len =
729 tokenize_printf_to_buffer!(&mut buffer, "Hello" PW_FMT_CONCAT " Pigweed").unwrap();
730 assert_eq!(&buffer[..len], &[0xe0, 0x92, 0xe0, 0xa]);
731
732 let len = tokenize_printf_to_buffer!(&mut buffer, "Hello: " PW_FMT_CONCAT "%cigweed",
733 "P".as_bytes()[0])
734 .unwrap();
735 assert_eq!(&buffer[..len], &[0x2e, 0x52, 0xac, 0xe4, 0xa0, 0x1]);
736 }
737
738 #[test]
739 fn tokenizer_supports_concatenated_core_fmt_format_strings() {
740 // Since the no argument and some arguments cases are handled differently
741 // by `tokenize_to_buffer!` we need to test both.
742 let mut buffer = [0u8; 64];
743 let len =
744 tokenize_core_fmt_to_buffer!(&mut buffer, "Hello" PW_FMT_CONCAT " Pigweed").unwrap();
745 assert_eq!(&buffer[..len], &[0xe0, 0x92, 0xe0, 0xa]);
746
747 let len = tokenize_core_fmt_to_buffer!(&mut buffer, "The answer is " PW_FMT_CONCAT "{}!",
748 1 as i32)
749 .unwrap();
750 assert_eq!(&buffer[..len], &[0x52, 0x1c, 0xb0, 0x4c, 0x2]);
751 }
752}