Skip to content

Commit 9f561eb

Browse files
AJIOBAJIOB
committed
Add support for preprocessed output
1 parent 760a109 commit 9f561eb

File tree

4 files changed

+266
-142
lines changed

4 files changed

+266
-142
lines changed

‎src/compiler/c.rs‎

Lines changed: 166 additions & 137 deletions
Original file line numberDiff line numberDiff line change
@@ -405,6 +405,8 @@ where
405405
);
406406
}
407407

408+
let support_preprocessing = self.parsed_args.language.support_preprocessing();
409+
408410
let use_preprocessor_cache_mode = {
409411
let can_use_preprocessor_cache_mode = preprocessor_cache_mode_config
410412
.use_preprocessor_cache_mode
@@ -431,6 +433,14 @@ where
431433
);
432434
}
433435

436+
if !support_preprocessing && use_preprocessor_cache_mode {
437+
debug!(
438+
"parse_arguments: Disabling preprocessor cache because language {} does not support preprocessing",
439+
self.parsed_args.language.as_str()
440+
);
441+
use_preprocessor_cache_mode = false;
442+
}
443+
434444
use_preprocessor_cache_mode
435445
};
436446

@@ -448,156 +458,167 @@ where
448458
} else {
449459
None
450460
};
451-
if let Some(preprocessor_key) = &preprocessor_key {
452-
if cache_control == CacheControl::Default {
453-
if let Some(mut seekable) = storage
454-
.get_preprocessor_cache_entry(preprocessor_key)
455-
.await?
456-
{
457-
let mut buf = vec![];
458-
seekable.read_to_end(&mut buf)?;
459-
let mut preprocessor_cache_entry = PreprocessorCacheEntry::read(&buf)?;
460-
let mut updated = false;
461-
let hit = preprocessor_cache_entry
462-
.lookup_result_digest(preprocessor_cache_mode_config, &mut updated);
463-
464-
let mut update_failed = false;
465-
if updated {
466-
// Time macros have been found, we need to update
467-
// the preprocessor cache entry. See [`PreprocessorCacheEntry::result_matches`].
468-
debug!(
469-
"Preprocessor cache updated because of time macros: {preprocessor_key}"
470-
);
471461

472-
if let Err(e) = storage
473-
.put_preprocessor_cache_entry(
474-
preprocessor_key,
475-
preprocessor_cache_entry,
476-
)
477-
.await
478-
{
479-
debug!("Failed to update preprocessor cache: {}", e);
480-
update_failed = true;
462+
let (preprocessor_stdout, include_files) = if support_preprocessing {
463+
if let Some(preprocessor_key) = &preprocessor_key {
464+
if cache_control == CacheControl::Default {
465+
if let Some(mut seekable) = storage
466+
.get_preprocessor_cache_entry(preprocessor_key)
467+
.await?
468+
{
469+
let mut buf = vec![];
470+
seekable.read_to_end(&mut buf)?;
471+
let mut preprocessor_cache_entry = PreprocessorCacheEntry::read(&buf)?;
472+
let mut updated = false;
473+
let hit = preprocessor_cache_entry
474+
.lookup_result_digest(preprocessor_cache_mode_config, &mut updated);
475+
476+
let mut update_failed = false;
477+
if updated {
478+
// Time macros have been found, we need to update
479+
// the preprocessor cache entry. See [`PreprocessorCacheEntry::result_matches`].
480+
debug!(
481+
"Preprocessor cache updated because of time macros: {preprocessor_key}"
482+
);
483+
484+
if let Err(e) = storage
485+
.put_preprocessor_cache_entry(
486+
preprocessor_key,
487+
preprocessor_cache_entry,
488+
)
489+
.await
490+
{
491+
debug!("Failed to update preprocessor cache: {}", e);
492+
update_failed = true;
493+
}
481494
}
482-
}
483495

484-
if !update_failed {
485-
if let Some(key) = hit {
486-
debug!("Preprocessor cache hit: {preprocessor_key}");
487-
// A compiler binary may be a symlink to another and
488-
// so has the same digest, but that means
489-
// the toolchain will not contain the correct path
490-
// to invoke the compiler! Add the compiler
491-
// executable path to try and prevent this
492-
let weak_toolchain_key = format!(
493-
"{}-{}",
494-
self.executable.to_string_lossy(),
495-
self.executable_digest
496-
);
497-
return Ok(HashResult {
498-
key,
499-
compilation: Box::new(CCompilation {
500-
parsed_args: self.parsed_args.to_owned(),
501-
is_locally_preprocessed: false,
502-
#[cfg(feature = "dist-client")]
503-
preprocessed_input: PREPROCESSING_SKIPPED_COMPILE_POISON
504-
.to_vec(),
505-
executable: self.executable.to_owned(),
506-
compiler: self.compiler.to_owned(),
507-
cwd: cwd.to_owned(),
508-
env_vars: env_vars.to_owned(),
509-
}),
510-
weak_toolchain_key,
511-
});
512-
} else {
513-
debug!("Preprocessor cache miss: {preprocessor_key}");
496+
if !update_failed {
497+
if let Some(key) = hit {
498+
debug!("Preprocessor cache hit: {preprocessor_key}");
499+
// A compiler binary may be a symlink to another and
500+
// so has the same digest, but that means
501+
// the toolchain will not contain the correct path
502+
// to invoke the compiler! Add the compiler
503+
// executable path to try and prevent this
504+
let weak_toolchain_key = format!(
505+
"{}-{}",
506+
self.executable.to_string_lossy(),
507+
self.executable_digest
508+
);
509+
return Ok(HashResult {
510+
key,
511+
compilation: Box::new(CCompilation {
512+
parsed_args: self.parsed_args.to_owned(),
513+
is_locally_preprocessed: false,
514+
#[cfg(feature = "dist-client")]
515+
preprocessed_input: PREPROCESSING_SKIPPED_COMPILE_POISON
516+
.to_vec(),
517+
executable: self.executable.to_owned(),
518+
compiler: self.compiler.to_owned(),
519+
cwd: cwd.to_owned(),
520+
env_vars: env_vars.to_owned(),
521+
}),
522+
weak_toolchain_key,
523+
});
524+
} else {
525+
debug!("Preprocessor cache miss: {preprocessor_key}");
526+
}
514527
}
515528
}
516529
}
517530
}
518-
}
519531

520-
let result = self
521-
.compiler
522-
.preprocess(
523-
creator,
524-
&self.executable,
525-
&self.parsed_args,
526-
&cwd,
527-
&env_vars,
528-
may_dist,
529-
rewrite_includes_only,
530-
use_preprocessor_cache_mode,
531-
)
532-
.await;
533-
let out_pretty = self.parsed_args.output_pretty().into_owned();
534-
let result = result.map_err(|e| {
535-
debug!("[{}]: preprocessor failed: {:?}", out_pretty, e);
536-
e
537-
});
532+
let result = self
533+
.compiler
534+
.preprocess(
535+
creator,
536+
&self.executable,
537+
&self.parsed_args,
538+
&cwd,
539+
&env_vars,
540+
may_dist,
541+
rewrite_includes_only,
542+
use_preprocessor_cache_mode,
543+
)
544+
.await;
545+
let out_pretty = self.parsed_args.output_pretty().into_owned();
546+
let result = result.map_err(|e| {
547+
debug!("[{}]: preprocessor failed: {:?}", out_pretty, e);
548+
e
549+
});
550+
551+
let outputs = self.parsed_args.outputs.clone();
552+
let args_cwd = cwd.clone();
553+
554+
let mut preprocessor_result = result.or_else(move |err| {
555+
// Errors remove all traces of potential output.
556+
debug!("removing files {:?}", &outputs);
557+
558+
let v: std::result::Result<(), std::io::Error> =
559+
outputs.values().try_for_each(|output| {
560+
let mut path = args_cwd.clone();
561+
path.push(&output.path);
562+
match fs::metadata(&path) {
563+
// File exists, remove it.
564+
Ok(_) => fs::remove_file(&path),
565+
_ => Ok(()),
566+
}
567+
});
568+
if v.is_err() {
569+
warn!("Could not remove files after preprocessing failed!");
570+
}
538571

539-
let outputs = self.parsed_args.outputs.clone();
540-
let args_cwd = cwd.clone();
541-
542-
let mut preprocessor_result = result.or_else(move |err| {
543-
// Errors remove all traces of potential output.
544-
debug!("removing files {:?}", &outputs);
545-
546-
let v: std::result::Result<(), std::io::Error> =
547-
outputs.values().try_for_each(|output| {
548-
let mut path = args_cwd.clone();
549-
path.push(&output.path);
550-
match fs::metadata(&path) {
551-
// File exists, remove it.
552-
Ok(_) => fs::remove_file(&path),
553-
_ => Ok(()),
572+
match err.downcast::<ProcessError>() {
573+
Ok(ProcessError(output)) => {
574+
debug!(
575+
"[{}]: preprocessor returned error status {:?}",
576+
out_pretty,
577+
output.status.code()
578+
);
579+
// Drop the stdout since it's the preprocessor output,
580+
// just hand back stderr and the exit status.
581+
bail!(ProcessError(process::Output {
582+
stdout: vec!(),
583+
..output
584+
}))
554585
}
555-
});
556-
if v.is_err() {
557-
warn!("Could not remove files after preprocessing failed!");
558-
}
586+
Err(err) => Err(err),
587+
}
588+
})?;
559589

560-
match err.downcast::<ProcessError>() {
561-
Ok(ProcessError(output)) => {
562-
debug!(
563-
"[{}]: preprocessor returned error status {:?}",
564-
out_pretty,
565-
output.status.code()
566-
);
567-
// Drop the stdout since it's the preprocessor output,
568-
// just hand back stderr and the exit status.
569-
bail!(ProcessError(process::Output {
570-
stdout: vec!(),
571-
..output
572-
}))
590+
// Remember include files needed in this preprocessing step
591+
let mut include_files = HashMap::new();
592+
if preprocessor_key.is_some() {
593+
// TODO how to propagate stats and which stats?
594+
if !process_preprocessed_file(
595+
&absolute_input_path,
596+
&cwd,
597+
&mut preprocessor_result.stdout,
598+
&mut include_files,
599+
preprocessor_cache_mode_config,
600+
start_of_compilation,
601+
StandardFsAbstraction,
602+
)? {
603+
debug!("Disabling preprocessor cache mode");
604+
preprocessor_key = None;
573605
}
574-
Err(err) => Err(err),
575606
}
576-
})?;
577607

578-
// Remember include files needed in this preprocessing step
579-
let mut include_files = HashMap::new();
580-
if preprocessor_key.is_some() {
581-
// TODO how to propagate stats and which stats?
582-
if !process_preprocessed_file(
583-
&absolute_input_path,
584-
&cwd,
585-
&mut preprocessor_result.stdout,
586-
&mut include_files,
587-
preprocessor_cache_mode_config,
588-
start_of_compilation,
589-
StandardFsAbstraction,
590-
)? {
591-
debug!("Disabling preprocessor cache mode");
592-
preprocessor_key = None;
593-
}
594-
}
608+
trace!(
609+
"[{}]: Preprocessor output is {} bytes",
610+
self.parsed_args.output_pretty(),
611+
preprocessor_result.stdout.len()
612+
);
595613

596-
trace!(
597-
"[{}]: Preprocessor output is {} bytes",
598-
self.parsed_args.output_pretty(),
599-
preprocessor_result.stdout.len()
600-
);
614+
(preprocessor_result.stdout, include_files)
615+
} else {
616+
// No preprocessing is supported - input is already preprocessed
617+
(
618+
std::fs::read(absolute_input_path.as_path())?,
619+
HashMap::new(),
620+
)
621+
};
601622

602623
// Create an argument vector containing both common and arch args, to
603624
// use in creating a hash key
@@ -611,7 +632,7 @@ where
611632
&common_and_arch_args,
612633
&extra_hashes,
613634
&env_vars,
614-
&preprocessor_result.stdout,
635+
&preprocessor_stdout,
615636
self.compiler.plusplus(),
616637
)
617638
};
@@ -650,7 +671,7 @@ where
650671
parsed_args: self.parsed_args.clone(),
651672
is_locally_preprocessed: true,
652673
#[cfg(feature = "dist-client")]
653-
preprocessed_input: preprocessor_result.stdout,
674+
preprocessed_input: preprocessor_stdout,
654675
executable: self.executable.clone(),
655676
compiler: self.compiler.clone(),
656677
cwd,
@@ -1640,6 +1661,8 @@ mod test {
16401661

16411662
t("c", Language::C);
16421663

1664+
t("i", Language::CPreprocessed);
1665+
16431666
t("C", Language::Cxx);
16441667
t("cc", Language::Cxx);
16451668
t("cp", Language::Cxx);
@@ -1648,6 +1671,8 @@ mod test {
16481671
t("cxx", Language::Cxx);
16491672
t("c++", Language::Cxx);
16501673

1674+
t("ii", Language::CxxPreprocessed);
1675+
16511676
t("h", Language::GenericHeader);
16521677

16531678
t("hh", Language::CxxHeader);
@@ -1661,9 +1686,13 @@ mod test {
16611686

16621687
t("m", Language::ObjectiveC);
16631688

1689+
t("mi", Language::ObjectiveCPreprocessed);
1690+
16641691
t("M", Language::ObjectiveCxx);
16651692
t("mm", Language::ObjectiveCxx);
16661693

1694+
t("mii", Language::ObjectiveCxxPreprocessed);
1695+
16671696
t("cu", Language::Cuda);
16681697
t("hip", Language::Hip);
16691698
}

‎src/compiler/clang.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -180,13 +180,18 @@ impl CCompilerImpl for Clang {
180180

181181
pub fn language_to_clang_arg(lang: Language) -> Option<&'static str> {
182182
match lang {
183+
// https://github.com/llvm/llvm-project/blob/main/clang/include/clang/Driver/Types.def
183184
Language::C => Some("c"),
184185
Language::CHeader => Some("c-header"),
186+
Language::CPreprocessed => Some("cpp-output"),
185187
Language::Cxx => Some("c++"),
186188
Language::CxxHeader => Some("c++-header"),
189+
Language::CxxPreprocessed => Some("c++-cpp-output"),
187190
Language::ObjectiveC => Some("objective-c"),
191+
Language::ObjectiveCPreprocessed => Some("objective-c-cpp-output"),
188192
Language::ObjectiveCxx => Some("objective-c++"),
189193
Language::ObjectiveCxxHeader => Some("objective-c++-header"),
194+
Language::ObjectiveCxxPreprocessed => Some("objective-c++-cpp-output"),
190195
Language::Cuda => Some("cuda"),
191196
Language::CudaFE => None,
192197
Language::Ptx => None,

0 commit comments

Comments
 (0)