pw_tokenizer/lib.rs
1// Copyright 2023 The Pigweed Authors
2//
3// Licensed under the Apache License, Version 2.0 (the "License"); you may not
4// use this file except in compliance with the License. You may obtain a copy of
5// the License at
6//
7// https://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12// License for the specific language governing permissions and limitations under
13// the License.
14
15//! `pw_tokenizer` - Efficient string handling and printf style encoding.
16//!
17//! Logging is critical, but developers are often forced to choose between
18//! additional logging or saving crucial flash space. The `pw_tokenizer` crate
19//! helps address this by replacing printf-style strings with binary tokens
20//! during compilation. This enables extensive logging with substantially less
21//! memory usage.
22//!
23//! For a more in depth explanation of the systems design and motivations,
24//! see [Pigweed's pw_tokenizer module documentation](https://pigweed.dev/pw_tokenizer/).
25//!
26//! # Examples
27//!
28//! Pigweed's tokenization database uses `printf` style strings internally so
29//! those are supported directly.
30//!
31//! ```
32//! use pw_tokenizer::tokenize_printf_to_buffer;
33//!
34//! let mut buffer = [0u8; 1024];
35//! let len = tokenize_printf_to_buffer!(&mut buffer, "The answer is %d", 42)?;
36//!
37//! // 4 bytes used to encode the token and one to encode the value 42. This
38//! // is a **3.5x** reduction in size compared to the raw string!
39//! assert_eq!(len, 5);
40//! # Ok::<(), pw_status::Error>(())
41//! ```
42//!
43//! We also support Rust's `core::fmt` style syntax. These format strings are
44//! converted to `printf` style at compile time to maintain compatibly with the
45//! rest of the Pigweed tokenizer ecosystem. The below example produces the
46//! same token and output as the above one.
47//!
48//! ```
49//! use pw_tokenizer::tokenize_core_fmt_to_buffer;
50//!
51//! let mut buffer = [0u8; 1024];
52//! let len = tokenize_core_fmt_to_buffer!(&mut buffer, "The answer is {}", 42 as i32)?;
53//! assert_eq!(len, 5);
54//! # Ok::<(), pw_status::Error>(())
55//! ```
56#![cfg_attr(not(feature = "std"), no_std)]
57#![deny(missing_docs)]
58
59use pw_status::Result;
60
61#[doc(hidden)]
62pub mod internal;
63
64/// Detokenization support.
65#[cfg(feature = "std")]
66pub mod detokenize;
67
68#[doc(hidden)]
69// Creating a __private namespace allows us a way to get to the modules
70// we need from macros by doing:
71// use $crate::__private as __pw_tokenizer_crate;
72//
73// This is how proc macro generated code can reliably reference back to
74// `pw_tokenizer` while still allowing a user to import it under a different
75// name.
76pub mod __private {
77 pub use pw_bytes::concat_static_strs;
78 pub use pw_format_core::{PrintfFormatter, PrintfHexFormatter, PrintfUpperHexFormatter};
79 pub use pw_status::Result;
80 pub use pw_stream::{Cursor, Seek, WriteInteger, WriteVarint};
81 pub use pw_tokenizer_core::hash_string;
82 #[cfg(feature = "proc_macro")]
83 pub use pw_tokenizer_macro::{
84 _token, _tokenize_core_fmt_to_buffer, _tokenize_core_fmt_to_writer,
85 _tokenize_printf_to_buffer, _tokenize_printf_to_writer,
86 };
87
88 pub use crate::*;
89}
90
91/// Return the [`u32`] token for the specified string and add it to the token
92/// database.
93///
94/// This is where the magic happens in `pw_tokenizer`! ... and by magic
95/// we mean hiding information in a special linker section that ends up in the
96/// final elf binary but does not get flashed to the device.
97///
98/// Two things are accomplished here:
99/// 1) The string is hashed into its stable `u32` token. This is the value that
100/// is returned from the macro.
101/// 2) A [token database entry](https://pigweed.dev/pw_tokenizer/design.html#binary-database-format)
102/// is generated, assigned to a unique static symbol, placed in a linker
103/// section named `pw_tokenizer.entries.<TOKEN_HASH>`. A
104/// [linker script](https://pigweed.googlesource.com/pigweed/pigweed/+/refs/heads/main/pw_tokenizer/pw_tokenizer_linker_sections.ld)
105/// is responsible for picking these symbols up and aggregating them into a
106/// single `.pw_tokenizer.entries` section in the final binary.
107///
108/// # Example
109/// ```
110/// use pw_tokenizer::token;
111///
112/// let token = token!("hello, \"world\"");
113/// assert_eq!(token, 3537412730);
114/// ```
115///
116/// Currently there is no support for encoding tokens to specific domains
117/// or with "fixed lengths" per [`pw_tokenizer_core::hash_bytes_fixed`].
118#[macro_export]
119macro_rules! token {
120 ($string:literal) => {{
121 use $crate::__private as __pw_tokenizer_crate;
122 $crate::__private::_token!($string)
123 }};
124}
125
126/// Tokenize a `core::fmt` style format string and arguments to an [`AsMut<u8>`]
127/// buffer. The format string is converted in to a `printf` and added token to
128/// the token database.
129///
130/// See [`token`] for an explanation on how strings are tokenized and entries
131/// are added to the token database. The token's domain is set to `""`.
132///
133/// Returns a [`pw_status::Result<usize>`] the number of bytes written to the buffer.
134///
135/// `tokenize_to_buffer!` supports concatenation of format strings as described
136/// in `pw_format::macros::FormatAndArgs`.
137///
138/// # Errors
139/// - [`pw_status::Error::OutOfRange`] - Buffer is not large enough to fit
140/// tokenized data.
141/// - [`pw_status::Error::InvalidArgument`] - Invalid buffer was provided.
142///
143/// # Example
144///
145/// ```
146/// use pw_tokenizer::tokenize_core_fmt_to_buffer;
147///
148/// // Tokenize a format string and argument into a buffer.
149/// let mut buffer = [0u8; 1024];
150/// let len = tokenize_core_fmt_to_buffer!(&mut buffer, "The answer is {}", 42 as i32)?;
151///
152/// // 4 bytes used to encode the token and one to encode the value 42.
153/// assert_eq!(len, 5);
154///
155/// // The format string can be composed of multiple strings literals using
156/// // the custom`PW_FMT_CONCAT` operator.
157/// let len = tokenize_core_fmt_to_buffer!(&mut buffer, "Hello " PW_FMT_CONCAT "Pigweed")?;
158///
159/// // Only a single 4 byte token is emitted after concatenation of the string
160/// // literals above.
161/// assert_eq!(len, 4);
162/// # Ok::<(), pw_status::Error>(())
163/// ```
164#[macro_export]
165macro_rules! tokenize_core_fmt_to_buffer {
166 ($buffer:expr, $($format_string:literal)PW_FMT_CONCAT+ $(, $args:expr)* $(,)?) => {{
167 use $crate::__private as __pw_tokenizer_crate;
168 __pw_tokenizer_crate::_tokenize_core_fmt_to_buffer!($buffer, $($format_string)PW_FMT_CONCAT+, $($args),*)
169 }};
170}
171
172/// Tokenize a printf format string and arguments to an [`AsMut<u8>`] buffer
173/// and add the format string's token to the token database.
174///
175/// See [`token`] for an explanation on how strings are tokenized and entries
176/// are added to the token database. The token's domain is set to `""`.
177///
178/// Returns a [`pw_status::Result<usize>`] the number of bytes written to the buffer.
179///
180/// `tokenize_to_buffer!` supports concatenation of format strings as described
181/// in `pw_format::macros::FormatAndArgs`.
182///
183/// # Errors
184/// - [`pw_status::Error::OutOfRange`] - Buffer is not large enough to fit
185/// tokenized data.
186/// - [`pw_status::Error::InvalidArgument`] - Invalid buffer was provided.
187///
188/// # Example
189///
190/// ```
191/// use pw_tokenizer::tokenize_printf_to_buffer;
192///
193/// // Tokenize a format string and argument into a buffer.
194/// let mut buffer = [0u8; 1024];
195/// let len = tokenize_printf_to_buffer!(&mut buffer, "The answer is %d", 42)?;
196///
197/// // 4 bytes used to encode the token and one to encode the value 42.
198/// assert_eq!(len, 5);
199///
200/// // The format string can be composed of multiple strings literals using
201/// // the custom`PW_FMT_CONCAT` operator.
202/// let len = tokenize_printf_to_buffer!(&mut buffer, "Hello " PW_FMT_CONCAT "Pigweed")?;
203///
204/// // Only a single 4 byte token is emitted after concatenation of the string
205/// // literals above.
206/// assert_eq!(len, 4);
207/// # Ok::<(), pw_status::Error>(())
208/// ```
209#[macro_export]
210macro_rules! tokenize_printf_to_buffer {
211 ($buffer:expr, $($format_string:literal)PW_FMT_CONCAT+ $(, $args:expr)* $(,)?) => {{
212 use $crate::__private as __pw_tokenizer_crate;
213 __pw_tokenizer_crate::_tokenize_printf_to_buffer!($buffer, $($format_string)PW_FMT_CONCAT+, $($args),*)
214 }};
215}
216
217/// Deprecated alias for [`tokenize_printf_to_buffer!`].
218#[macro_export]
219macro_rules! tokenize_to_buffer {
220 ($buffer:expr, $($format_string:literal)PW_FMT_CONCAT+ $(, $args:expr)* $(,)?) => {{
221 $crate::tokenize_printf_to_buffer!($buffer, $($format_string)PW_FMT_CONCAT+, $($args),*)
222 }};
223}
224
225/// Tokenize a `core::fmt` format string and arguments to a [`MessageWriter`].
226/// The format string is converted in to a `printf` and added token to the token
227/// database.
228///
229/// `tokenize_core_fmt_to_writer!` and the accompanying [`MessageWriter`] trait
230/// provide an optimized API for use cases like logging where the output of the
231/// tokenization will be written to a shared/ambient resource like stdio, a
232/// UART, or a shared buffer.
233///
234/// The `writer_type` should implement [`MessageWriter`] and [`Default`] traits.
235/// The writer is instantiated with the [`Default`] allowing any intermediate
236/// buffers to be declared on the stack of the internal writing engine instead
237/// of the caller's stack.
238///
239/// See [`token`] for an explanation on how strings are tokenized and entries
240/// are added to the token database. The token's domain is set to `""`.
241///
242/// Returns a [`pw_status::Result<()>`].
243///
244/// `tokenize_core_fmt_to_writer!` supports concatenation of format strings as
245/// described in `pw_format::macros::FormatAndArgs`.
246///
247/// # Errors
248/// - [`pw_status::Error::OutOfRange`] - [`MessageWriter`] does not have enough
249/// space to fit tokenized data.
250/// - others - `tokenize_core_fmt_to_writer!` will pass on any errors returned
251/// by the [`MessageWriter`].
252///
253/// # Code Size
254///
255/// This data was collected by examining the disassembly of a test program
256/// built for a Cortex M0.
257///
258/// | Tokenized Message | Per Call-site Cost (bytes) |
259/// | --------------------| -------------------------- |
260/// | no arguments | 10 |
261/// | one `i32` argument | 18 |
262///
263/// # Example
264///
265/// ```
266/// use pw_status::Result;
267/// use pw_stream::{Cursor, Write};
268/// use pw_tokenizer::{MessageWriter, tokenize_core_fmt_to_writer};
269///
270/// const BUFFER_LEN: usize = 32;
271///
272/// // Declare a simple MessageWriter that uses a [`pw_status::Cursor`] to
273/// // maintain an internal buffer.
274/// struct TestMessageWriter {
275/// cursor: Cursor<[u8; BUFFER_LEN]>,
276/// }
277///
278/// impl Default for TestMessageWriter {
279/// fn default() -> Self {
280/// Self {
281/// cursor: Cursor::new([0u8; BUFFER_LEN]),
282/// }
283/// }
284/// }
285///
286/// impl MessageWriter for TestMessageWriter {
287/// fn write(&mut self, data: &[u8]) -> Result<()> {
288/// self.cursor.write_all(data)
289/// }
290///
291/// fn remaining(&self) -> usize {
292/// self.cursor.remaining()
293/// }
294///
295/// fn finalize(self) -> Result<()> {
296/// let len = self.cursor.position();
297/// // 4 bytes used to encode the token and one to encode the value 42.
298/// assert_eq!(len, 5);
299/// Ok(())
300/// }
301/// }
302///
303/// // Tokenize a format string and argument into the writer. Note how we
304/// // pass in the message writer's type, not an instance of it.
305/// let len = tokenize_core_fmt_to_writer!(TestMessageWriter, "The answer is {}", 42 as i32)?;
306/// # Ok::<(), pw_status::Error>(())
307/// ```
308#[macro_export]
309macro_rules! tokenize_core_fmt_to_writer {
310 ($writer:expr, $($format_string:literal)PW_FMT_CONCAT+ $(, $args:expr)* $(,)?) => {{
311 use $crate::__private as __pw_tokenizer_crate;
312 __pw_tokenizer_crate::_tokenize_core_fmt_to_writer!($writer, $($format_string)PW_FMT_CONCAT+, $($args),*)
313 }};
314}
315
316/// Tokenize a `printf` format string and arguments to a [`MessageWriter`] and
317/// add the format string's token to the token database.
318///
319/// `tokenize_printf_fmt_to_writer!` and the accompanying [`MessageWriter`] trait
320/// provide an optimized API for use cases like logging where the output of the
321/// tokenization will be written to a shared/ambient resource like stdio, a
322/// UART, or a shared buffer.
323///
324/// The `writer_type` should implement [`MessageWriter`] and [`Default`] traits.
325/// The writer is instantiated with the [`Default`] allowing any intermediate
326/// buffers to be declared on the stack of the internal writing engine instead
327/// of the caller's stack.
328///
329/// See [`token`] for an explanation on how strings are tokenized and entries
330/// are added to the token database. The token's domain is set to `""`.
331///
332/// Returns a [`pw_status::Result<()>`].
333///
334/// `tokenize_core_fmt_to_writer!` supports concatenation of format strings as
335/// described in `pw_format::macros::FormatAndArgs`.
336///
337/// # Errors
338/// - [`pw_status::Error::OutOfRange`] - [`MessageWriter`] does not have enough
339/// space to fit tokenized data.
340/// - others - `tokenize_printf_to_writer!` will pass on any errors returned
341/// by the [`MessageWriter`].
342///
343/// # Code Size
344///
345/// This data was collected by examining the disassembly of a test program
346/// built for a Cortex M0.
347///
348/// | Tokenized Message | Per Call-site Cost (bytes) |
349/// | --------------------| -------------------------- |
350/// | no arguments | 10 |
351/// | one `i32` argument | 18 |
352///
353/// # Example
354///
355/// ```
356/// use pw_status::Result;
357/// use pw_stream::{Cursor, Write};
358/// use pw_tokenizer::{MessageWriter, tokenize_printf_to_writer};
359///
360/// const BUFFER_LEN: usize = 32;
361///
362/// // Declare a simple MessageWriter that uses a [`pw_status::Cursor`] to
363/// // maintain an internal buffer.
364/// struct TestMessageWriter {
365/// cursor: Cursor<[u8; BUFFER_LEN]>,
366/// }
367///
368/// impl Default for TestMessageWriter {
369/// fn default() -> Self {
370/// Self {
371/// cursor: Cursor::new([0u8; BUFFER_LEN]),
372/// }
373/// }
374/// }
375///
376/// impl MessageWriter for TestMessageWriter {
377/// fn write(&mut self, data: &[u8]) -> Result<()> {
378/// self.cursor.write_all(data)
379/// }
380///
381/// fn remaining(&self) -> usize {
382/// self.cursor.remaining()
383/// }
384///
385/// fn finalize(self) -> Result<()> {
386/// let len = self.cursor.position();
387/// // 4 bytes used to encode the token and one to encode the value 42.
388/// assert_eq!(len, 5);
389/// Ok(())
390/// }
391/// }
392///
393/// // Tokenize a format string and argument into the writer. Note how we
394/// // pass in the message writer's type, not an instance of it.
395/// let len = tokenize_printf_to_writer!(TestMessageWriter, "The answer is %d", 42)?;
396/// # Ok::<(), pw_status::Error>(())
397/// ```
398#[macro_export]
399macro_rules! tokenize_printf_to_writer {
400 ($writer:expr, $($format_string:literal)PW_FMT_CONCAT+ $(, $args:expr)* $(,)?) => {{
401 use $crate::__private as __pw_tokenizer_crate;
402 __pw_tokenizer_crate::_tokenize_printf_to_writer!($writer, $($format_string)PW_FMT_CONCAT+, $($args),*)
403 }};
404}
405
406/// Deprecated alias for [`tokenize_printf_to_writer!`].
407#[macro_export]
408macro_rules! tokenize_to_writer {
409 ($writer:expr, $($format_string:literal)PW_FMT_CONCAT+ $(, $args:expr)* $(,)?) => {{
410 $crate::tokenize_printf_to_writer!($writer, $($format_string)PW_FMT_CONCAT+, $($args),*)
411 }};
412}
413
414/// A trait used by [`tokenize_to_writer!`] to output tokenized messages.
415///
416/// For more details on how this type is used, see the [`tokenize_to_writer!`]
417/// documentation.
418pub trait MessageWriter {
419 /// Append `data` to the message.
420 fn write(&mut self, data: &[u8]) -> Result<()>;
421
422 /// Return the remaining space in this message instance.
423 ///
424 /// If there are no space constraints, return `usize::MAX`.
425 fn remaining(&self) -> usize;
426
427 /// Finalize message.
428 ///
429 /// `finalize()` is called when the tokenized message is complete.
430 fn finalize(self) -> Result<()>;
431}
432
433#[cfg(test)]
434// Untyped prints code rely on as casts to annotate type information.
435#[allow(clippy::unnecessary_cast)]
436#[allow(clippy::literal_string_with_formatting_args)]
437mod tests {
438 use super::*;
439 extern crate self as pw_tokenizer;
440 use core::cell::RefCell;
441
442 use pw_stream::{Cursor, Write};
443
444 // This is not meant to be an exhaustive test of tokenization which is
445 // covered by `pw_tokenizer_core`'s unit tests. Rather, this is testing
446 // that the `tokenize!` macro connects to that correctly.
447 #[test]
448 fn test_token() {}
449
450 macro_rules! tokenize_to_buffer_test {
451 ($expected_data:expr, $buffer_len:expr, $printf_fmt:literal, $core_fmt:literal $(, $args:expr)* $(,)?) => {{
452 if $printf_fmt != "" {
453 let mut buffer = [0u8; $buffer_len];
454 let len = tokenize_printf_to_buffer!(&mut buffer, $printf_fmt, $($args),*).unwrap();
455 assert_eq!(
456 &buffer[..len],
457 $expected_data,
458 "printf style input does not produce expected output",
459 );
460 }
461 if $core_fmt != "" {
462 let mut buffer = [0u8; $buffer_len];
463 let len = tokenize_core_fmt_to_buffer!(&mut buffer, $core_fmt, $($args),*).unwrap();
464 assert_eq!(
465 &buffer[..len],
466 $expected_data,
467 "core::fmt style input does not produce expected output",
468 );
469 }
470 }}
471 }
472
473 macro_rules! tokenize_to_writer_test {
474 ($expected_data:expr, $buffer_len:expr, $printf_fmt:literal, $core_fmt:literal $(, $args:expr)* $(,)?) => {{
475 // The `MessageWriter` API is used in places like logging where it
476 // accesses an shared/ambient resource (like stdio or an UART). To test
477 // it in a hermetic way we declare test specific `MessageWriter` that
478 // writes it's output to a scoped static variable that can be checked
479 // after the test is run.
480
481 // Since these tests are not multi-threaded, we can use a thread_local!
482 // instead of a mutex.
483 thread_local!(static TEST_OUTPUT: RefCell<Option<Vec<u8>>> = RefCell::new(None));
484
485 struct TestMessageWriter {
486 cursor: Cursor<[u8; $buffer_len]>,
487 }
488
489 impl Default for TestMessageWriter {
490 fn default() -> Self {
491 Self {
492 cursor: Cursor::new([0u8; $buffer_len]),
493 }
494 }
495 }
496
497 impl MessageWriter for TestMessageWriter {
498 fn write(&mut self, data: &[u8]) -> Result<()> {
499 self.cursor.write_all(data)
500 }
501
502 fn remaining(&self) -> usize {
503 self.cursor.remaining()
504 }
505
506 fn finalize(self) -> Result<()> {
507 let write_len = self.cursor.position();
508 let data = self.cursor.into_inner();
509 TEST_OUTPUT.with(|output| *output.borrow_mut() = Some(data[..write_len].to_vec()));
510
511 Ok(())
512 }
513 }
514
515 if $printf_fmt != "" {
516 TEST_OUTPUT.with(|output| *output.borrow_mut() = None);
517 tokenize_printf_to_writer!(TestMessageWriter, $printf_fmt, $($args),*).unwrap();
518 TEST_OUTPUT.with(|output| {
519 assert_eq!(
520 *output.borrow(),
521 Some($expected_data.to_vec()),
522 )
523 });
524 }
525
526 if $core_fmt != "" {
527 TEST_OUTPUT.with(|output| *output.borrow_mut() = None);
528 tokenize_core_fmt_to_writer!(TestMessageWriter, $core_fmt, $($args),*).unwrap();
529 TEST_OUTPUT.with(|output| {
530 assert_eq!(
531 *output.borrow(),
532 Some($expected_data.to_vec()),
533 )
534 });
535 }
536 }}
537 }
538
539 macro_rules! tokenize_test {
540 ($expected_data:expr, $buffer_len:expr, $printf_fmt:literal, $core_fmt:literal $(, $args:expr)* $(,)?) => {{
541 tokenize_to_buffer_test!($expected_data, $buffer_len, $printf_fmt, $core_fmt, $($args),*);
542 tokenize_to_writer_test!($expected_data, $buffer_len, $printf_fmt, $core_fmt, $($args),*);
543 }};
544 }
545
546 #[test]
547 fn bare_string_encodes_correctly() {
548 tokenize_test!(
549 &[0xe0, 0x92, 0xe0, 0xa], // expected buffer
550 64, // buffer size
551 "Hello Pigweed", // printf style
552 "Hello Pigweed", // core::fmt style
553 );
554 }
555
556 #[test]
557 fn test_decimal_format() {
558 // "as casts" are used for the integer arguments below. They are only
559 // need for the core::fmt style arguments but are added so that we can
560 // check that the printf and core::fmt style equivalents encode the same.
561 tokenize_test!(
562 &[0x52, 0x1c, 0xb0, 0x4c, 0x2], // expected buffer
563 64, // buffer size
564 "The answer is %d!", // printf style
565 "The answer is {}!", // core::fmt style
566 1 as i32
567 );
568
569 tokenize_test!(
570 &[0x36, 0xd0, 0xfb, 0x69, 0x1], // expected buffer
571 64, // buffer size
572 "No! The answer is %d!", // printf style
573 "No! The answer is {}!", // core::fmt style
574 -1 as i32
575 );
576
577 tokenize_test!(
578 &[0xa4, 0xad, 0x50, 0x54, 0x0], // expected buffer
579 64, // buffer size
580 "I think you'll find that the answer is %d!", // printf style
581 "I think you'll find that the answer is {}!", // core::fmt style
582 0 as i32
583 );
584 }
585
586 #[test]
587 fn test_misc_integer_format() {
588 // %d, %i, %o, %u, %x, %X all encode integers the same.
589 tokenize_test!(
590 &[0x52, 0x1c, 0xb0, 0x4c, 0x2], // expected buffer
591 64, // buffer size
592 "The answer is %d!", // printf style
593 "", // no equivalent core::fmt style
594 1
595 );
596
597 // Because %i is an alias for %d, it gets converted to a %d by the
598 // `pw_format` macro infrastructure.
599 tokenize_test!(
600 &[0x52, 0x1c, 0xb0, 0x4c, 0x2], // expected buffer
601 64, // buffer size
602 "The answer is %i!", // printf style
603 "", // no equivalent core::fmt style
604 1
605 );
606
607 tokenize_test!(
608 &[0x5d, 0x70, 0x12, 0xb4, 0x2], // expected buffer
609 64, // buffer size
610 "The answer is %o!", // printf style
611 "", // no equivalent core::fmt style
612 1u32
613 );
614
615 tokenize_test!(
616 &[0x63, 0x58, 0x5f, 0x8f, 0x2], // expected buffer
617 64, // buffer size
618 "The answer is %u!", // printf style
619 "", // no equivalent core::fmt style
620 1u32
621 );
622
623 tokenize_test!(
624 &[0x66, 0xcc, 0x05, 0x7d, 0x2], // expected buffer
625 64, // buffer size
626 "The answer is %x!", // printf style
627 "", // no equivalent core::fmt style
628 1u32
629 );
630
631 tokenize_test!(
632 &[0x46, 0x4c, 0x16, 0x96, 0x2], // expected buffer
633 64, // buffer size
634 "The answer is %X!", // printf style
635 "", // no equivalent core::fmt style
636 1u32
637 );
638 }
639
640 #[test]
641 fn test_string_format() {
642 tokenize_test!(
643 b"\x25\xf6\x2e\x66\x07Pigweed", // expected buffer
644 64, // buffer size
645 "Hello: %s!", // printf style
646 "", // no equivalent core::fmt style
647 "Pigweed"
648 );
649 }
650
651 #[test]
652 fn test_string_format_overflow() {
653 tokenize_test!(
654 b"\x25\xf6\x2e\x66\x83Pig", // expected buffer
655 8, // buffer size
656 "Hello: %s!", // printf style
657 "", // no equivalent core::fmt style
658 "Pigweed"
659 );
660 }
661
662 #[test]
663 fn test_char_format() {
664 tokenize_test!(
665 &[0x2e, 0x52, 0xac, 0xe4, 0xa0, 0x1], // expected buffer
666 64, // buffer size
667 "Hello: %cigweed", // printf style
668 "", // no equivalent core::fmt style
669 "P".as_bytes()[0]
670 );
671 }
672
673 #[test]
674 fn test_untyped_format() {
675 tokenize_test!(
676 &[0x63, 0x58, 0x5f, 0x8f, 0x2], // expected buffer
677 64, // buffer size
678 "The answer is %u!", // printf style
679 "The answer is {}!", // core::fmt style
680 1 as u32
681 );
682
683 tokenize_test!(
684 &[0x36, 0xd0, 0xfb, 0x69, 0x1], // expected buffer
685 64, // buffer size
686 "No! The answer is %v!", // printf style
687 "No! The answer is {}!", // core::fmt style
688 -1 as i32
689 );
690
691 tokenize_test!(
692 b"\x25\xf6\x2e\x66\x07Pigweed", // expected buffer
693 64, // buffer size
694 "Hello: %v!", // printf style
695 "Hello: {}!", // core::fmt style
696 "Pigweed" as &str
697 );
698 }
699
700 #[test]
701 fn test_field_width_and_zero_pad_format() {
702 tokenize_test!(
703 &[0x3a, 0xc2, 0x1a, 0x05, 0xfc, 0xab, 0x06], // expected buffer
704 64, // buffer size
705 "Lets go to the %x", // printf style
706 "Lets go to the {:x}", // core::fmt style
707 0xcafe as u32
708 );
709
710 tokenize_test!(
711 &[0xf3, 0x16, 0x03, 0x99, 0xfc, 0xab, 0x06], // expected buffer
712 64, // buffer size
713 "Lets go to the %8x", // printf style
714 "Lets go to the {:8x}", // core::fmt style
715 0xcafe as u32
716 );
717
718 tokenize_test!(
719 &[0x44, 0xce, 0xa3, 0x7e, 0xfc, 0xab, 0x06], // expected buffer
720 64, // buffer size
721 "Lets go to the %08x", // printf style
722 "Lets go to the {:08x}", // core::fmt style
723 0xcafe as u32
724 );
725 }
726
727 #[test]
728 fn tokenizer_supports_concatenated_printf_format_strings() {
729 // Since the no argument and some arguments cases are handled differently
730 // by `tokenize_to_buffer!` we need to test both.
731 let mut buffer = [0u8; 64];
732 let len =
733 tokenize_printf_to_buffer!(&mut buffer, "Hello" PW_FMT_CONCAT " Pigweed").unwrap();
734 assert_eq!(&buffer[..len], &[0xe0, 0x92, 0xe0, 0xa]);
735
736 let len = tokenize_printf_to_buffer!(&mut buffer, "Hello: " PW_FMT_CONCAT "%cigweed",
737 "P".as_bytes()[0])
738 .unwrap();
739 assert_eq!(&buffer[..len], &[0x2e, 0x52, 0xac, 0xe4, 0xa0, 0x1]);
740 }
741
742 #[test]
743 fn tokenizer_supports_concatenated_core_fmt_format_strings() {
744 // Since the no argument and some arguments cases are handled differently
745 // by `tokenize_to_buffer!` we need to test both.
746 let mut buffer = [0u8; 64];
747 let len =
748 tokenize_core_fmt_to_buffer!(&mut buffer, "Hello" PW_FMT_CONCAT " Pigweed").unwrap();
749 assert_eq!(&buffer[..len], &[0xe0, 0x92, 0xe0, 0xa]);
750
751 let len = tokenize_core_fmt_to_buffer!(&mut buffer, "The answer is " PW_FMT_CONCAT "{}!",
752 1 as i32)
753 .unwrap();
754 assert_eq!(&buffer[..len], &[0x52, 0x1c, 0xb0, 0x4c, 0x2]);
755 }
756}