prost_build/lib.rs
1#![doc(html_root_url = "https://docs.rs/prost-build/0.13.5")]
2#![allow(clippy::option_as_ref_deref, clippy::format_push_string)]
3
4//! `prost-build` compiles `.proto` files into Rust.
5//!
6//! `prost-build` is designed to be used for build-time code generation as part of a Cargo
7//! build-script.
8//!
9//! ## Example
10//!
11//! Let's create a small library crate, `snazzy`, that defines a collection of
12//! snazzy new items in a protobuf file.
13//!
14//! ```bash
15//! $ cargo new --lib snazzy && cd snazzy
16//! ```
17//!
18//! First, add `prost-build` and `prost` as dependencies to `Cargo.toml`:
19//!
20//! ```bash
21//! $ cargo add --build prost-build
22//! $ cargo add prost
23//! ```
24//!
25//! Next, add `src/items.proto` to the project:
26//!
27//! ```proto
28//! syntax = "proto3";
29//!
30//! package snazzy.items;
31//!
32//! // A snazzy new shirt!
33//! message Shirt {
34//! // Label sizes
35//! enum Size {
36//! SMALL = 0;
37//! MEDIUM = 1;
38//! LARGE = 2;
39//! }
40//!
41//! // The base color
42//! string color = 1;
43//! // The size as stated on the label
44//! Size size = 2;
45//! }
46//! ```
47//!
48//! To generate Rust code from `items.proto`, we use `prost-build` in the crate's
49//! `build.rs` build-script:
50//!
51//! ```rust,no_run
52//! use std::io::Result;
53//! fn main() -> Result<()> {
54//! prost_build::compile_protos(&["src/items.proto"], &["src/"])?;
55//! Ok(())
56//! }
57//! ```
58//!
59//! And finally, in `lib.rs`, include the generated code:
60//!
61//! ```rust,ignore
62//! // Include the `items` module, which is generated from items.proto.
63//! // It is important to maintain the same structure as in the proto.
64//! pub mod snazzy {
65//! pub mod items {
66//! include!(concat!(env!("OUT_DIR"), "/snazzy.items.rs"));
67//! }
68//! }
69//!
70//! use snazzy::items;
71//!
72//! /// Returns a large shirt of the specified color
73//! pub fn create_large_shirt(color: String) -> items::Shirt {
74//! let mut shirt = items::Shirt::default();
75//! shirt.color = color;
76//! shirt.set_size(items::shirt::Size::Large);
77//! shirt
78//! }
79//! ```
80//!
81//! That's it! Run `cargo doc` to see documentation for the generated code. The full
82//! example project can be found on [GitHub](https://github.com/danburkert/snazzy).
83//!
84//! ## Feature Flags
85//! - `format`: Format the generated output. This feature is enabled by default.
86//! - `cleanup-markdown`: Clean up Markdown in protobuf docs. Enable this to clean up protobuf files from third parties.
87//!
88//! ### Cleaning up Markdown in code docs
89//!
90//! If you are using protobuf files from third parties, where the author of the protobuf
91//! is not treating comments as Markdown, or is, but has codeblocks in their docs,
92//! then you may need to clean up the documentation in order that `cargo test --doc`
93//! will not fail spuriously, and that `cargo doc` doesn't attempt to render the
94//! codeblocks as Rust code.
95//!
96//! To do this, in your `Cargo.toml`, add `features = ["cleanup-markdown"]` to the inclusion
97//! of the `prost-build` crate and when your code is generated, the code docs will automatically
98//! be cleaned up a bit.
99//!
100//! ## Sourcing `protoc`
101//!
102//! `prost-build` depends on the Protocol Buffers compiler, `protoc`, to parse `.proto` files into
103//! a representation that can be transformed into Rust.
104//!
105//! The easiest way for `prost-build` to find `protoc` is to install it in your `PATH`.
106//! This can be done by following the [`protoc` install instructions]. `prost-build` will search
107//! the current path for `protoc` or `protoc.exe`.
108//!
109//! When `protoc` is installed in a different location, set `PROTOC` to the path of the executable.
110//! If set, `prost-build` uses the `PROTOC`
111//! for locating `protoc`. For example, on a macOS system where Protobuf is installed
112//! with Homebrew, set the environment variables to:
113//!
114//! ```bash
115//! PROTOC=/usr/local/bin/protoc
116//! ```
117//!
118//! Alternatively, the path to `protoc` execuatable can be explicitly set
119//! via [`Config::protoc_executable()`].
120//!
121//! If `prost-build` can not find `protoc`
122//! via these methods the `compile_protos` method will fail.
123//!
124//! [`protoc` install instructions]: https://github.com/protocolbuffers/protobuf#protocol-compiler-installation
125//!
126//! ### Compiling `protoc` from source
127//!
128//! To compile `protoc` from source you can use the `protobuf-src` crate and
129//! set the correct environment variables.
130//! ```no_run,ignore, rust
131//! std::env::set_var("PROTOC", protobuf_src::protoc());
132//!
133//! // Now compile your proto files via prost-build
134//! ```
135//!
136//! [`protobuf-src`]: https://docs.rs/protobuf-src
137
138use std::io::Result;
139use std::path::Path;
140
141use prost_types::FileDescriptorSet;
142
143mod ast;
144pub use crate::ast::{Comments, Method, Service};
145
146mod collections;
147pub(crate) use collections::{BytesType, MapType};
148
149mod code_generator;
150mod context;
151mod extern_paths;
152mod ident;
153mod message_graph;
154mod path;
155
156mod config;
157pub use config::{
158 error_message_protoc_not_found, protoc_from_env, protoc_include_from_env, Config,
159};
160
161mod module;
162pub use module::Module;
163
164/// A service generator takes a service descriptor and generates Rust code.
165///
166/// `ServiceGenerator` can be used to generate application-specific interfaces
167/// or implementations for Protobuf service definitions.
168///
169/// Service generators are registered with a code generator using the
170/// `Config::service_generator` method.
171///
172/// A viable scenario is that an RPC framework provides a service generator. It generates a trait
173/// describing methods of the service and some glue code to call the methods of the trait, defining
174/// details like how errors are handled or if it is asynchronous. Then the user provides an
175/// implementation of the generated trait in the application code and plugs it into the framework.
176///
177/// Such framework isn't part of Prost at present.
178pub trait ServiceGenerator {
179 /// Generates a Rust interface or implementation for a service, writing the
180 /// result to `buf`.
181 fn generate(&mut self, service: Service, buf: &mut String);
182
183 /// Finalizes the generation process.
184 ///
185 /// In case there's something that needs to be output at the end of the generation process, it
186 /// goes here. Similar to [`generate`](Self::generate), the output should be appended to
187 /// `buf`.
188 ///
189 /// An example can be a module or other thing that needs to appear just once, not for each
190 /// service generated.
191 ///
192 /// This still can be called multiple times in a lifetime of the service generator, because it
193 /// is called once per `.proto` file.
194 ///
195 /// The default implementation is empty and does nothing.
196 fn finalize(&mut self, _buf: &mut String) {}
197
198 /// Finalizes the generation process for an entire protobuf package.
199 ///
200 /// This differs from [`finalize`](Self::finalize) by where (and how often) it is called
201 /// during the service generator life cycle. This method is called once per protobuf package,
202 /// making it ideal for grouping services within a single package spread across multiple
203 /// `.proto` files.
204 ///
205 /// The default implementation is empty and does nothing.
206 fn finalize_package(&mut self, _package: &str, _buf: &mut String) {}
207}
208
209/// Compile `.proto` files into Rust files during a Cargo build.
210///
211/// The generated `.rs` files are written to the Cargo `OUT_DIR` directory, suitable for use with
212/// the [include!][1] macro. See the [Cargo `build.rs` code generation][2] example for more info.
213///
214/// This function should be called in a project's `build.rs`.
215///
216/// # Arguments
217///
218/// **`protos`** - Paths to `.proto` files to compile. Any transitively [imported][3] `.proto`
219/// files are automatically be included.
220///
221/// **`includes`** - Paths to directories in which to search for imports. Directories are searched
222/// in order. The `.proto` files passed in **`protos`** must be found in one of the provided
223/// include directories.
224///
225/// # Errors
226///
227/// This function can fail for a number of reasons:
228///
229/// - Failure to locate or download `protoc`.
230/// - Failure to parse the `.proto`s.
231/// - Failure to locate an imported `.proto`.
232/// - Failure to compile a `.proto` without a [package specifier][4].
233///
234/// It's expected that this function call be `unwrap`ed in a `build.rs`; there is typically no
235/// reason to gracefully recover from errors during a build.
236///
237/// # Example `build.rs`
238///
239/// ```rust,no_run
240/// # use std::io::Result;
241/// fn main() -> Result<()> {
242/// prost_build::compile_protos(&["src/frontend.proto", "src/backend.proto"], &["src"])?;
243/// Ok(())
244/// }
245/// ```
246///
247/// [1]: https://doc.rust-lang.org/std/macro.include.html
248/// [2]: http://doc.crates.io/build-script.html#case-study-code-generation
249/// [3]: https://developers.google.com/protocol-buffers/docs/proto3#importing-definitions
250/// [4]: https://developers.google.com/protocol-buffers/docs/proto#packages
251pub fn compile_protos(protos: &[impl AsRef<Path>], includes: &[impl AsRef<Path>]) -> Result<()> {
252 Config::new().compile_protos(protos, includes)
253}
254
255/// Compile a [`FileDescriptorSet`] into Rust files during a Cargo build.
256///
257/// The generated `.rs` files are written to the Cargo `OUT_DIR` directory, suitable for use with
258/// the [include!][1] macro. See the [Cargo `build.rs` code generation][2] example for more info.
259///
260/// This function should be called in a project's `build.rs`.
261///
262/// This function can be combined with a crate like [`protox`] which outputs a
263/// [`FileDescriptorSet`] and is a pure Rust implementation of `protoc`.
264///
265/// # Example
266/// ```rust,no_run
267/// # use prost_types::FileDescriptorSet;
268/// # fn fds() -> FileDescriptorSet { todo!() }
269/// fn main() -> std::io::Result<()> {
270/// let file_descriptor_set = fds();
271///
272/// prost_build::compile_fds(file_descriptor_set)
273/// }
274/// ```
275///
276/// [`protox`]: https://github.com/andrewhickman/protox
277/// [1]: https://doc.rust-lang.org/std/macro.include.html
278/// [2]: http://doc.crates.io/build-script.html#case-study-code-generation
279pub fn compile_fds(fds: FileDescriptorSet) -> Result<()> {
280 Config::new().compile_fds(fds)
281}
282
283#[cfg(test)]
284mod tests {
285 use std::cell::RefCell;
286 use std::fs::File;
287 use std::io::Read;
288 use std::rc::Rc;
289
290 use super::*;
291
292 macro_rules! assert_eq_fixture_file {
293 ($expected_path:expr, $actual_path:expr) => {{
294 let actual = std::fs::read_to_string($actual_path).unwrap();
295
296 // Normalizes windows and Linux-style EOL
297 let actual = actual.replace("\r\n", "\n");
298
299 assert_eq_fixture_contents!($expected_path, actual);
300 }};
301 }
302
303 macro_rules! assert_eq_fixture_contents {
304 ($expected_path:expr, $actual:expr) => {{
305 let expected = std::fs::read_to_string($expected_path).unwrap();
306
307 // Normalizes windows and Linux-style EOL
308 let expected = expected.replace("\r\n", "\n");
309
310 if expected != $actual {
311 std::fs::write($expected_path, &$actual).unwrap();
312 }
313
314 assert_eq!(expected, $actual);
315 }};
316 }
317
318 /// An example service generator that generates a trait with methods corresponding to the
319 /// service methods.
320 struct ServiceTraitGenerator;
321
322 impl ServiceGenerator for ServiceTraitGenerator {
323 fn generate(&mut self, service: Service, buf: &mut String) {
324 // Generate a trait for the service.
325 service.comments.append_with_indent(0, buf);
326 buf.push_str(&format!("trait {} {{\n", &service.name));
327
328 // Generate the service methods.
329 for method in service.methods {
330 method.comments.append_with_indent(1, buf);
331 buf.push_str(&format!(
332 " fn {}(_: {}) -> {};\n",
333 method.name, method.input_type, method.output_type
334 ));
335 }
336
337 // Close out the trait.
338 buf.push_str("}\n");
339 }
340 fn finalize(&mut self, buf: &mut String) {
341 // Needs to be present only once, no matter how many services there are
342 buf.push_str("pub mod utils { }\n");
343 }
344 }
345
346 /// Implements `ServiceGenerator` and provides some state for assertions.
347 struct MockServiceGenerator {
348 state: Rc<RefCell<MockState>>,
349 }
350
351 /// Holds state for `MockServiceGenerator`
352 #[derive(Default)]
353 struct MockState {
354 service_names: Vec<String>,
355 package_names: Vec<String>,
356 finalized: u32,
357 }
358
359 impl MockServiceGenerator {
360 fn new(state: Rc<RefCell<MockState>>) -> Self {
361 Self { state }
362 }
363 }
364
365 impl ServiceGenerator for MockServiceGenerator {
366 fn generate(&mut self, service: Service, _buf: &mut String) {
367 let mut state = self.state.borrow_mut();
368 state.service_names.push(service.name);
369 }
370
371 fn finalize(&mut self, _buf: &mut String) {
372 let mut state = self.state.borrow_mut();
373 state.finalized += 1;
374 }
375
376 fn finalize_package(&mut self, package: &str, _buf: &mut String) {
377 let mut state = self.state.borrow_mut();
378 state.package_names.push(package.to_string());
379 }
380 }
381
382 #[test]
383 fn smoke_test() {
384 let _ = env_logger::try_init();
385 let tempdir = tempfile::tempdir().unwrap();
386
387 Config::new()
388 .service_generator(Box::new(ServiceTraitGenerator))
389 .out_dir(tempdir.path())
390 .compile_protos(&["src/fixtures/smoke_test/smoke_test.proto"], &["src"])
391 .unwrap();
392 }
393
394 #[test]
395 fn finalize_package() {
396 let _ = env_logger::try_init();
397 let tempdir = tempfile::tempdir().unwrap();
398
399 let state = Rc::new(RefCell::new(MockState::default()));
400 let gen = MockServiceGenerator::new(Rc::clone(&state));
401
402 Config::new()
403 .service_generator(Box::new(gen))
404 .include_file("_protos.rs")
405 .out_dir(tempdir.path())
406 .compile_protos(
407 &[
408 "src/fixtures/helloworld/hello.proto",
409 "src/fixtures/helloworld/goodbye.proto",
410 ],
411 &["src/fixtures/helloworld"],
412 )
413 .unwrap();
414
415 let state = state.borrow();
416 assert_eq!(&state.service_names, &["Greeting", "Farewell"]);
417 assert_eq!(&state.package_names, &["helloworld"]);
418 assert_eq!(state.finalized, 3);
419 }
420
421 #[test]
422 fn test_generate_message_attributes() {
423 let _ = env_logger::try_init();
424 let tempdir = tempfile::tempdir().unwrap();
425
426 let mut config = Config::new();
427 config
428 .out_dir(tempdir.path())
429 // Add attributes to all messages and enums
430 .message_attribute(".", "#[derive(derive_builder::Builder)]")
431 .enum_attribute(".", "#[some_enum_attr(u8)]");
432
433 let fds = config
434 .load_fds(
435 &["src/fixtures/helloworld/hello.proto"],
436 &["src/fixtures/helloworld"],
437 )
438 .unwrap();
439
440 // Add custom attributes to messages that are service inputs or outputs.
441 for file in &fds.file {
442 for service in &file.service {
443 for method in &service.method {
444 if let Some(input) = &method.input_type {
445 config.message_attribute(input, "#[derive(custom_proto::Input)]");
446 }
447 if let Some(output) = &method.output_type {
448 config.message_attribute(output, "#[derive(custom_proto::Output)]");
449 }
450 }
451 }
452 }
453
454 config.compile_fds(fds).unwrap();
455
456 assert_eq_fixture_file!(
457 if cfg!(feature = "format") {
458 "src/fixtures/helloworld/_expected_helloworld_formatted.rs"
459 } else {
460 "src/fixtures/helloworld/_expected_helloworld.rs"
461 },
462 tempdir.path().join("helloworld.rs")
463 );
464 }
465
466 #[test]
467 fn test_generate_no_empty_outputs() {
468 let _ = env_logger::try_init();
469 let state = Rc::new(RefCell::new(MockState::default()));
470 let gen = MockServiceGenerator::new(Rc::clone(&state));
471 let include_file = "_include.rs";
472 let tempdir = tempfile::tempdir().unwrap();
473 let previously_empty_proto_path = tempdir.path().join(Path::new("google.protobuf.rs"));
474
475 Config::new()
476 .service_generator(Box::new(gen))
477 .include_file(include_file)
478 .out_dir(tempdir.path())
479 .compile_protos(
480 &["src/fixtures/imports_empty/imports_empty.proto"],
481 &["src/fixtures/imports_empty"],
482 )
483 .unwrap();
484
485 // Prior to PR introducing this test, the generated include file would have the file
486 // google.protobuf.rs which was an empty file. Now that file should only exist if it has content
487 if let Ok(mut f) = File::open(previously_empty_proto_path) {
488 // Since this file was generated, it should not be empty.
489 let mut contents = String::new();
490 f.read_to_string(&mut contents).unwrap();
491 assert!(!contents.is_empty());
492 } else {
493 // The file wasn't generated so the result include file should not reference it
494 assert_eq_fixture_file!(
495 "src/fixtures/imports_empty/_expected_include.rs",
496 tempdir.path().join(Path::new(include_file))
497 );
498 }
499 }
500
501 #[test]
502 fn test_generate_field_attributes() {
503 let _ = env_logger::try_init();
504 let tempdir = tempfile::tempdir().unwrap();
505
506 Config::new()
507 .out_dir(tempdir.path())
508 .boxed("Container.data.foo")
509 .boxed("Bar.qux")
510 .compile_protos(
511 &["src/fixtures/field_attributes/field_attributes.proto"],
512 &["src/fixtures/field_attributes"],
513 )
514 .unwrap();
515
516 assert_eq_fixture_file!(
517 if cfg!(feature = "format") {
518 "src/fixtures/field_attributes/_expected_field_attributes_formatted.rs"
519 } else {
520 "src/fixtures/field_attributes/_expected_field_attributes.rs"
521 },
522 tempdir.path().join("field_attributes.rs")
523 );
524 }
525
526 #[test]
527 fn deterministic_include_file() {
528 let _ = env_logger::try_init();
529
530 for _ in 1..10 {
531 let state = Rc::new(RefCell::new(MockState::default()));
532 let gen = MockServiceGenerator::new(Rc::clone(&state));
533 let include_file = "_include.rs";
534 let tempdir = tempfile::tempdir().unwrap();
535
536 Config::new()
537 .service_generator(Box::new(gen))
538 .include_file(include_file)
539 .out_dir(tempdir.path())
540 .compile_protos(
541 &[
542 "src/fixtures/alphabet/a.proto",
543 "src/fixtures/alphabet/b.proto",
544 "src/fixtures/alphabet/c.proto",
545 "src/fixtures/alphabet/d.proto",
546 "src/fixtures/alphabet/e.proto",
547 "src/fixtures/alphabet/f.proto",
548 ],
549 &["src/fixtures/alphabet"],
550 )
551 .unwrap();
552
553 assert_eq_fixture_file!(
554 "src/fixtures/alphabet/_expected_include.rs",
555 tempdir.path().join(Path::new(include_file))
556 );
557 }
558 }
559
560 #[test]
561 fn write_includes() {
562 let modules = [
563 Module::from_protobuf_package_name("foo.bar.baz"),
564 Module::from_protobuf_package_name(""),
565 Module::from_protobuf_package_name("foo.bar"),
566 Module::from_protobuf_package_name("bar"),
567 Module::from_protobuf_package_name("foo"),
568 Module::from_protobuf_package_name("foo.bar.qux"),
569 Module::from_protobuf_package_name("foo.bar.a.b.c"),
570 ];
571
572 let file_names = modules
573 .iter()
574 .map(|m| (m.clone(), m.to_file_name_or("_.default")))
575 .collect();
576
577 let mut buf = Vec::new();
578 Config::new()
579 .default_package_filename("_.default")
580 .write_includes(modules.iter().collect(), &mut buf, None, &file_names)
581 .unwrap();
582 let actual = String::from_utf8(buf).unwrap();
583 assert_eq_fixture_contents!("src/fixtures/write_includes/_.includes.rs", actual);
584 }
585}