From 40b630bcd2e94813c7c08ee4c39806a40122fb9e Mon Sep 17 00:00:00 2001 From: Toshiki Teramura Date: Tue, 23 Jun 2020 12:48:45 +0900 Subject: [PATCH 1/5] Remove unused --- intel-mkl-tool/src/lib.rs | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/intel-mkl-tool/src/lib.rs b/intel-mkl-tool/src/lib.rs index 8e5c546d..31264153 100644 --- a/intel-mkl-tool/src/lib.rs +++ b/intel-mkl-tool/src/lib.rs @@ -91,7 +91,6 @@ //! use anyhow::*; -use log::*; use std::path::*; mod config; @@ -149,19 +148,3 @@ pub fn xdg_home_path() -> PathBuf { mkl::VERSION_UPDATE )) } - -pub fn seek_pkg_config() -> Option { - if let Ok(lib) = pkg_config::probe_library("mkl-dynamic-lp64-seq") { - if lib.libs.len() > 1 { - warn!("Found {} MKL libraries. Use first found.", lib.libs.len()) - } - return Some(PathBuf::from(lib.libs[0].clone())); - } - None -} - -pub fn download_default>(out_dir: P) -> Result<()> { - let cfg = Config::from_str("mkl-dynamic-lp64-seq").unwrap(); - cfg.download(out_dir)?; - Ok(()) -} From ba7f0a010311002498ab4a8e938c5f83bc66dda9 Mon Sep 17 00:00:00 2001 From: Toshiki Teramura Date: Tue, 23 Jun 2020 12:50:56 +0900 Subject: [PATCH 2/5] Remove log and env_logger --- intel-mkl-tool/Cargo.toml | 4 +--- intel-mkl-tool/src/cli.rs | 10 +++------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/intel-mkl-tool/Cargo.toml b/intel-mkl-tool/Cargo.toml index 9dacca15..9483ee1f 100644 --- a/intel-mkl-tool/Cargo.toml +++ b/intel-mkl-tool/Cargo.toml @@ -11,7 +11,7 @@ license = "MIT" [features] default = ["cli"] -cli = ["structopt", "env_logger"] +cli = ["structopt"] [dependencies] anyhow = "1.0.31" @@ -19,14 +19,12 @@ curl = "0.4.29" derive_more = "0.99.8" dirs = "2.0.2" glob = "0.3.0" -log = "0.4.8" pkg-config = "0.3.17" tar = "0.4.29" zstd = "0.5.3" # CLI structopt = { version = "0.3.15", optional = true } -env_logger = { version = "0.7.1", optional = true } [dev-dependencies] paste = "0.1.17" diff --git a/intel-mkl-tool/src/cli.rs b/intel-mkl-tool/src/cli.rs index 7d267119..4c5ce4d7 100644 --- a/intel-mkl-tool/src/cli.rs +++ b/intel-mkl-tool/src/cli.rs @@ -1,6 +1,5 @@ use anyhow::*; use intel_mkl_tool::*; -use log::*; use std::{env, path::PathBuf}; use structopt::StructOpt; @@ -34,9 +33,6 @@ enum Opt { } fn main() -> Result<()> { - env::set_var("RUST_LOG", "info"); - env_logger::init(); - let opt = Opt::from_args(); match opt { @@ -47,7 +43,7 @@ fn main() -> Result<()> { cfg.download(&path.join(cfg.name()))?; } else { for cfg in Config::possibles() { - info!( + println!( "Download archive {:<22} into {}", cfg.name(), path.display() @@ -85,7 +81,7 @@ fn main() -> Result<()> { path }; let package = entry.package(&path)?; - info!("Pacakge created: {}", package.display()); + println!("Pacakge created: {}", package.display()); } else { for entry in Entry::available() { let path = if let Ok(version) = entry.version() { @@ -94,7 +90,7 @@ fn main() -> Result<()> { path.clone() }; let package = entry.package(&path)?; - info!("Pacakge created: {}", package.display()); + println!("Pacakge created: {}", package.display()); } } } From 671abbc01603d4e31c01d47ec041ef42ce8c2a3e Mon Sep 17 00:00:00 2001 From: Toshiki Teramura Date: Tue, 23 Jun 2020 13:32:23 +0900 Subject: [PATCH 3/5] Split archive feature --- intel-mkl-tool/Cargo.toml | 13 +++-- intel-mkl-tool/src/config.rs | 82 ------------------------------- intel-mkl-tool/src/download.rs | 90 ++++++++++++++++++++++++++++++++++ intel-mkl-tool/src/entry.rs | 20 -------- intel-mkl-tool/src/lib.rs | 7 +++ intel-mkl-tool/src/package.rs | 24 +++++++++ 6 files changed, 129 insertions(+), 107 deletions(-) create mode 100644 intel-mkl-tool/src/download.rs create mode 100644 intel-mkl-tool/src/package.rs diff --git a/intel-mkl-tool/Cargo.toml b/intel-mkl-tool/Cargo.toml index 9483ee1f..990d29eb 100644 --- a/intel-mkl-tool/Cargo.toml +++ b/intel-mkl-tool/Cargo.toml @@ -11,20 +11,23 @@ license = "MIT" [features] default = ["cli"] -cli = ["structopt"] +archive = ["curl", "tar", "zstd"] +cli = ["structopt", "archive"] [dependencies] anyhow = "1.0.31" -curl = "0.4.29" derive_more = "0.99.8" dirs = "2.0.2" glob = "0.3.0" pkg-config = "0.3.17" -tar = "0.4.29" -zstd = "0.5.3" + +# archive +curl = { version = "0.4.29", optional = true } +tar = { version = "0.4.29", optional = true } +zstd = { version = "0.5.3", optional = true } # CLI -structopt = { version = "0.3.15", optional = true } +structopt = { version = "0.3.15", optional = true } [dev-dependencies] paste = "0.1.17" diff --git a/intel-mkl-tool/src/config.rs b/intel-mkl-tool/src/config.rs index fc9db5a1..8729c5ba 100644 --- a/intel-mkl-tool/src/config.rs +++ b/intel-mkl-tool/src/config.rs @@ -1,7 +1,5 @@ use crate::*; -use curl::easy::Easy; use derive_more::*; -use std::fs; pub const VALID_CONFIGS: &[&str] = &[ "mkl-dynamic-ilp64-iomp", @@ -146,41 +144,6 @@ impl Config { } } } - - /// Download archive from AWS S3, and expand into `${out_dir}/*.so` - pub fn download>(&self, out_dir: P) -> Result<()> { - let out_dir = out_dir.as_ref(); - if out_dir.exists() { - fs::create_dir_all(&out_dir)?; - } - let data = read_from_url(&format!("{}/{}.tar.zst", s3_addr(), self.name()))?; - let zstd = zstd::stream::read::Decoder::new(data.as_slice())?; - let mut arc = tar::Archive::new(zstd); - arc.unpack(&out_dir)?; - Ok(()) - } -} - -/// Helper for download file from URL -/// -/// - This function expands obtained data into memory space -/// -fn read_from_url(url: &str) -> Result> { - let mut data = Vec::new(); - let mut handle = Easy::new(); - handle.fail_on_error(true)?; - handle.url(url)?; - { - let mut transfer = handle.transfer(); - transfer - .write_function(|new_data| { - data.extend_from_slice(new_data); - Ok(new_data.len()) - }) - .unwrap(); - transfer.perform().unwrap(); - } - Ok(data) } #[cfg(test)] @@ -220,49 +183,4 @@ mod tests { assert!(Config::from_str("mkl-static-lp64-omp").is_err()); Ok(()) } - - macro_rules! impl_test_download { - ($name:expr) => { - paste::item! { - #[test] - fn []() -> Result<()> { - let name = $name; - let cfg = Config::from_str(name)?; - cfg.download(format!("test_download/{}", name))?; - Ok(()) - } - } - }; - } - - #[cfg(target_os = "windows")] - mod macos { - use super::*; - impl_test_download!("mkl-dynamic-lp64-seq"); - impl_test_download!("mkl-dynamic-ilp64-seq"); - impl_test_download!("mkl-static-lp64-seq"); - impl_test_download!("mkl-static-ilp64-seq"); - } - - #[cfg(target_os = "macos")] - mod macos { - use super::*; - impl_test_download!("mkl-dynamic-lp64-seq"); - impl_test_download!("mkl-dynamic-lp64-iomp"); - impl_test_download!("mkl-dynamic-ilp64-seq"); - impl_test_download!("mkl-dynamic-ilp64-iomp"); - } - - #[cfg(target_os = "linux")] - mod linux { - use super::*; - impl_test_download!("mkl-dynamic-lp64-seq"); - impl_test_download!("mkl-dynamic-lp64-iomp"); - impl_test_download!("mkl-dynamic-ilp64-seq"); - impl_test_download!("mkl-dynamic-ilp64-iomp"); - impl_test_download!("mkl-static-lp64-seq"); - impl_test_download!("mkl-static-lp64-iomp"); - impl_test_download!("mkl-static-ilp64-seq"); - impl_test_download!("mkl-static-ilp64-iomp"); - } } diff --git a/intel-mkl-tool/src/download.rs b/intel-mkl-tool/src/download.rs new file mode 100644 index 00000000..1188f527 --- /dev/null +++ b/intel-mkl-tool/src/download.rs @@ -0,0 +1,90 @@ +use crate::*; +use curl::easy::Easy; +use std::fs; + +impl Config { + /// Download archive from AWS S3, and expand into `${out_dir}/*.so` + pub fn download>(&self, out_dir: P) -> Result<()> { + let out_dir = out_dir.as_ref(); + if out_dir.exists() { + fs::create_dir_all(&out_dir)?; + } + let data = read_from_url(&format!("{}/{}.tar.zst", s3_addr(), self.name()))?; + let zstd = zstd::stream::read::Decoder::new(data.as_slice())?; + let mut arc = tar::Archive::new(zstd); + arc.unpack(&out_dir)?; + Ok(()) + } +} + +/// Helper for download file from URL +/// +/// - This function expands obtained data into memory space +/// +fn read_from_url(url: &str) -> Result> { + let mut data = Vec::new(); + let mut handle = Easy::new(); + handle.fail_on_error(true)?; + handle.url(url)?; + { + let mut transfer = handle.transfer(); + transfer + .write_function(|new_data| { + data.extend_from_slice(new_data); + Ok(new_data.len()) + }) + .unwrap(); + transfer.perform().unwrap(); + } + Ok(data) +} + +#[cfg(test)] +mod tests { + use super::*; + + macro_rules! impl_test_download { + ($name:expr) => { + paste::item! { + #[test] + fn []() -> Result<()> { + let name = $name; + let cfg = Config::from_str(name)?; + cfg.download(format!("test_download/{}", name))?; + Ok(()) + } + } + }; + } + + #[cfg(target_os = "windows")] + mod macos { + use super::*; + impl_test_download!("mkl-dynamic-lp64-seq"); + impl_test_download!("mkl-dynamic-ilp64-seq"); + impl_test_download!("mkl-static-lp64-seq"); + impl_test_download!("mkl-static-ilp64-seq"); + } + + #[cfg(target_os = "macos")] + mod macos { + use super::*; + impl_test_download!("mkl-dynamic-lp64-seq"); + impl_test_download!("mkl-dynamic-lp64-iomp"); + impl_test_download!("mkl-dynamic-ilp64-seq"); + impl_test_download!("mkl-dynamic-ilp64-iomp"); + } + + #[cfg(target_os = "linux")] + mod linux { + use super::*; + impl_test_download!("mkl-dynamic-lp64-seq"); + impl_test_download!("mkl-dynamic-lp64-iomp"); + impl_test_download!("mkl-dynamic-ilp64-seq"); + impl_test_download!("mkl-dynamic-ilp64-iomp"); + impl_test_download!("mkl-static-lp64-seq"); + impl_test_download!("mkl-static-lp64-iomp"); + impl_test_download!("mkl-static-ilp64-seq"); + impl_test_download!("mkl-static-ilp64-iomp"); + } +} diff --git a/intel-mkl-tool/src/entry.rs b/intel-mkl-tool/src/entry.rs index 155fad46..6885bb07 100644 --- a/intel-mkl-tool/src/entry.rs +++ b/intel-mkl-tool/src/entry.rs @@ -193,26 +193,6 @@ impl Entry { bail!("Cannot determine MKL versions"); } - pub fn package(&self, out_dir: &Path) -> Result { - fs::create_dir_all(out_dir)?; - let out = out_dir.join(format!("{}.tar.zst", self.name())); - if out.exists() { - bail!("Output archive already exits: {}", out.display()); - } - let f = fs::File::create(&out)?; - let buf = io::BufWriter::new(f); - let zstd = zstd::stream::write::Encoder::new(buf, 6)?; - let mut ar = tar::Builder::new(zstd); - ar.mode(tar::HeaderMode::Deterministic); - for (path, name) in self.found_files() { - let lib = path.join(&name); - ar.append_path_with_name(lib, name)?; - } - let zstd = ar.into_inner()?; - zstd.finish()?; - Ok(out) - } - pub fn print_cargo_metadata(&self) { let paths: HashSet = self .found_files() diff --git a/intel-mkl-tool/src/lib.rs b/intel-mkl-tool/src/lib.rs index 31264153..49785d12 100644 --- a/intel-mkl-tool/src/lib.rs +++ b/intel-mkl-tool/src/lib.rs @@ -90,12 +90,19 @@ //! |mkl_cdft_core | libmkl_cdft_core.so| libmkl_cdft_core.a| //! +#![cfg_attr(not(feature = "archive"), allow(dead_code))] + use anyhow::*; use std::path::*; mod config; mod entry; +#[cfg(feature = "archive")] +mod download; +#[cfg(feature = "archive")] +mod package; + pub use config::*; pub use entry::*; diff --git a/intel-mkl-tool/src/package.rs b/intel-mkl-tool/src/package.rs new file mode 100644 index 00000000..41a9efcb --- /dev/null +++ b/intel-mkl-tool/src/package.rs @@ -0,0 +1,24 @@ +use crate::*; +use std::{fs, io}; + +impl Entry { + pub fn package(&self, out_dir: &Path) -> Result { + fs::create_dir_all(out_dir)?; + let out = out_dir.join(format!("{}.tar.zst", self.name())); + if out.exists() { + bail!("Output archive already exits: {}", out.display()); + } + let f = fs::File::create(&out)?; + let buf = io::BufWriter::new(f); + let zstd = zstd::stream::write::Encoder::new(buf, 6)?; + let mut ar = tar::Builder::new(zstd); + ar.mode(tar::HeaderMode::Deterministic); + for (path, name) in self.found_files() { + let lib = path.join(&name); + ar.append_path_with_name(lib, name)?; + } + let zstd = ar.into_inner()?; + zstd.finish()?; + Ok(out) + } +} From 3cbdad352b098db2be783ab40c89f010f65fc4f1 Mon Sep 17 00:00:00 2001 From: Toshiki Teramura Date: Tue, 23 Jun 2020 13:49:15 +0900 Subject: [PATCH 4/5] Make download optional --- intel-mkl-src/Cargo.toml | 4 ++-- intel-mkl-src/build.rs | 9 +++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/intel-mkl-src/Cargo.toml b/intel-mkl-src/Cargo.toml index d62ee563..7f49faee 100644 --- a/intel-mkl-src/Cargo.toml +++ b/intel-mkl-src/Cargo.toml @@ -27,8 +27,8 @@ mkl-dynamic-lp64-seq = [] mkl-dynamic-ilp64-iomp = [] mkl-dynamic-ilp64-seq = [] -# Enable downloading from AWS S3 when not found -download = [] +# Enable downloading from AWS S3 when system MKL not found +download = ["intel-mkl-tool/archive"] # (Experimental) Cache download archive ad $XDG_DATA_HOME/intel-mkl-tool/ xdg-data-home = [] diff --git a/intel-mkl-src/build.rs b/intel-mkl-src/build.rs index dad6aa97..cb30b575 100644 --- a/intel-mkl-src/build.rs +++ b/intel-mkl-src/build.rs @@ -20,6 +20,8 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. +#![cfg_attr(feature = "download", allow(unreachable_code))] + use anyhow::*; use intel_mkl_tool::*; use std::{env, path::*}; @@ -51,7 +53,8 @@ fn main() -> Result<()> { } // download if not found - if cfg!(feature = "download") { + #[cfg(feature = "download")] + { let path = if cfg!(feature = "xdg-data-home") { xdg_home_path() } else { @@ -64,6 +67,8 @@ fn main() -> Result<()> { cfg.download(path)?; let entry = Entry::from_config(cfg).unwrap(); // must found entry.print_cargo_metadata(); + return Ok(()); } - Ok(()) + + bail!("No MKL found, and download flag is off."); } From 925c35a414a35c9b308ff36f6fa8c016a0a17aa5 Mon Sep 17 00:00:00 2001 From: Toshiki Teramura Date: Tue, 23 Jun 2020 13:58:56 +0900 Subject: [PATCH 5/5] cargo-test with no default feature --- .github/workflows/intel-mkl-tool.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/intel-mkl-tool.yml b/.github/workflows/intel-mkl-tool.yml index 3679a656..4af9f472 100644 --- a/.github/workflows/intel-mkl-tool.yml +++ b/.github/workflows/intel-mkl-tool.yml @@ -22,6 +22,12 @@ jobs: with: command: test args: --manifest-path=intel-mkl-tool/Cargo.toml + name: cargo-test + - uses: actions-rs/cargo@v1 + with: + command: test + args: --manifest-path=intel-mkl-tool/Cargo.toml --no-default-features + name: cargo-test no-default-features docker: runs-on: ubuntu-18.04