From 7b86ef4ffed696de92fcd87d2f0cf607778a295a Mon Sep 17 00:00:00 2001 From: Rob Williamson Date: Wed, 1 Jan 2025 10:58:44 +0100 Subject: [PATCH 01/13] Correct Warnings Especially the unsafe use of NodeHandles and the test logger. --- src/server/websocket/socket.rs | 23 ++++--- src/server/websocket/use_websocket.rs | 2 +- src/test.rs | 92 +++++++++++++-------------- 3 files changed, 62 insertions(+), 55 deletions(-) diff --git a/src/server/websocket/socket.rs b/src/server/websocket/socket.rs index 8a79de0..89e6ed5 100644 --- a/src/server/websocket/socket.rs +++ b/src/server/websocket/socket.rs @@ -1,6 +1,8 @@ use crate::core::node_handles::NodeHandles; #[cfg(test)] use crate::server::websocket::result::ResultRejection; +#[cfg(test)] +use crate::server::websocket::socket::tests::HANDLES; #[cfg(not(test))] use crate::server::websocket::websocket_handler::handler; #[cfg(test)] @@ -22,8 +24,9 @@ pub(crate) async fn handler(ws: Ws, handles: NodeHandles) -> ResultRejection ResultRejection = None; + lazy_static! { + pub(super) static ref HANDLES: Arc>> = Arc::new(Mutex::new(None)); + } mod websocket { use super::*; @@ -41,8 +48,9 @@ mod tests { #[tokio::test] async fn calls_handler() { - unsafe { - HANDLES = None; + { + let mut lock = HANDLES.lock().unwrap(); + *lock = None; } let route = websocket(&get_test_node_handles()); @@ -53,9 +61,8 @@ mod tests { .await .expect("handshake"); - unsafe { - assert!(HANDLES.is_some(), "handles not set"); - } + let lock = HANDLES.lock().unwrap(); + assert!(lock.is_some(), "handles not set"); } } } diff --git a/src/server/websocket/use_websocket.rs b/src/server/websocket/use_websocket.rs index 33c6a95..4f99973 100644 --- a/src/server/websocket/use_websocket.rs +++ b/src/server/websocket/use_websocket.rs @@ -272,7 +272,7 @@ mod tests { } } - mod BreakInstructionTranslator { + mod break_instruction_translator { use super::*; use crate::server::websocket::use_websocket::BreakInstructionTranslator; mod none_or_break_parent { diff --git a/src/test.rs b/src/test.rs index b0f0f30..69ebcd0 100644 --- a/src/test.rs +++ b/src/test.rs @@ -4,11 +4,13 @@ pub(crate) use crate::test::tests::LogEntry; pub(crate) use crate::test::tests::Logger; #[cfg(test)] pub(crate) mod tests { + use lazy_static::lazy_static; use log::{Log, Metadata, Record}; use parking_lot::ReentrantMutex; use std::cell::RefCell; use std::future::Future; - use std::sync::{Arc, Mutex, MutexGuard, Once}; + use std::ops::Deref; + use std::sync::{Arc, MutexGuard, Once}; #[derive(Clone, Debug, PartialEq)] pub(crate) struct LogEntry { @@ -37,52 +39,18 @@ pub(crate) mod tests { } } - static mut LOGGER: Option = None; - - pub(crate) struct Logger { + pub(crate) struct LogAccessor { records: Arc>>>, } - impl Logger { - fn get() -> &'static mut Logger { - unsafe { - static mut LOGGER_INIT: Mutex<()> = Mutex::new(()); - let _guard = LOGGER_INIT.lock().unwrap(); - - match LOGGER { - Some(ref mut l) => l, - None => { - LOGGER = Some(Logger::new()); - LOGGER.as_mut().unwrap() - } - } - } - } - - pub(crate) fn init() { - static INIT: Once = Once::new(); - INIT.call_once(|| { - log::set_logger(Logger::get()).unwrap(); - log::set_max_level(log::LevelFilter::Trace); - }); - } - - pub(crate) fn use_in(log_use: T) - where - T: FnOnce(&mut Logger) -> (), - { - let mutex = Logger::get().records.clone(); - let _lock = mutex.lock(); - log_use(Logger::get()); - } - - pub(crate) fn new() -> Self { + impl LogAccessor { + fn new() -> Self { Self { records: Arc::new(ReentrantMutex::from(RefCell::new(Vec::new()))), } } - pub(crate) fn log_entries(&mut self) -> Vec { + pub(crate) fn log_entries(&self) -> Vec { self.records .lock() .borrow() @@ -100,23 +68,55 @@ pub(crate) mod tests { exists } - pub(crate) fn reset(&mut self) { + pub(crate) fn reset(&self) { self.records.lock().borrow_mut().clear(); } } + lazy_static! { + static ref LOGGER: Logger = Logger::new(); + } + + pub(crate) struct Logger { + log_accessor: Arc, + } + + impl Logger { + pub(crate) fn init() { + static INIT: Once = Once::new(); + INIT.call_once(|| { + log::set_logger(LOGGER.deref()).unwrap(); + log::set_max_level(log::LevelFilter::Trace); + }); + } + + fn new() -> Self { + Self { + log_accessor: Arc::new(LogAccessor::new()), + } + } + + pub(crate) fn use_in(log_use: T) + where + T: FnOnce(&LogAccessor) -> (), + { + let _guard = LOGGER.log_accessor.records.lock(); + let log_accessor = LOGGER.log_accessor.clone(); + log_use(log_accessor.as_ref()); + } + } + impl Log for Logger { fn enabled(&self, _metadata: &Metadata) -> bool { true } fn log(&self, record: &Record) { - unsafe { - (*self.records.lock().as_ptr()).push(LogEntry::new( - record.args().to_string().as_str(), - record.level(), - )); - } + let lock = self.log_accessor.records.lock(); + lock.borrow_mut().push(LogEntry::new( + record.args().to_string().as_str(), + record.level(), + )); } fn flush(&self) {} From 85dc6640151b16e56779023fc234016222c23956 Mon Sep 17 00:00:00 2001 From: Rob Williamson Date: Wed, 1 Jan 2025 19:31:34 +0100 Subject: [PATCH 02/13] Retry After Initialized Because otherwise we're waiting for another init signal that will never come. --- src/integration/google/source.rs | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/src/integration/google/source.rs b/src/integration/google/source.rs index 963309c..6b4e8c7 100644 --- a/src/integration/google/source.rs +++ b/src/integration/google/source.rs @@ -50,20 +50,30 @@ impl Source { let task = task::spawn(async move { drop(permit); + let mut initialized = false; loop { - match load_receiver.recv().await { - Some(_) => { - info!("Loading google source"); + if !initialized { + match load_receiver.recv().await { + Some(_) => { + info!("Loading google source"); + initialized = true; + } + None => { + error!("Channel closed"); + break; + } } - None => { - error!("Channel closed"); + } else { + if load_receiver.is_closed() { break; } } + let application_secret = if let Some(config) = get_config().await { config.to_application_secret(&core_config) } else { + Self::wait_in_loop().await; continue; }; let token_path = match get_token_path::().await { @@ -93,7 +103,7 @@ impl Source { } }; sync(delegate).await; - sleep(Duration::from_secs(10)).await; + Self::wait_in_loop().await; } } }); @@ -164,4 +174,8 @@ impl Source { let (_task_result, _stop_result) = join!(task, lifetime_task); } + + async fn wait_in_loop() { + sleep(Duration::from_secs(10)).await; + } } From b50fe7d135cd1e11169568009e1d92e9072bfcc5 Mon Sep 17 00:00:00 2001 From: Rob Williamson Date: Wed, 1 Jan 2025 22:12:01 +0100 Subject: [PATCH 03/13] Factor out the Client To make it possible to write a test client. --- src/domain/oauth2/client.rs | 53 ++++++++++++++++--------- src/domain/oauth2/mod.rs | 3 +- src/integration/google/auth/delegate.rs | 21 +++++++--- src/integration/google/source.rs | 4 +- src/integration/google/tasks/sync.rs | 6 ++- 5 files changed, 59 insertions(+), 28 deletions(-) diff --git a/src/domain/oauth2/client.rs b/src/domain/oauth2/client.rs index afaab19..efbf133 100644 --- a/src/domain/oauth2/client.rs +++ b/src/domain/oauth2/client.rs @@ -17,6 +17,7 @@ use oauth2::{ AccessToken, AuthorizationRequest, CsrfToken, PkceCodeChallenge, PkceCodeVerifier, RefreshToken, Scope, }; +use std::future::Future; use std::path::{Path, PathBuf}; use std::sync::Arc; use tokio::fs; @@ -27,8 +28,22 @@ use tokio::time::sleep; use Error::Oauth2CsrfMismatch; use Event::Oauth2Code; +pub trait Client: Clone + Send + Sized + Sync + 'static { + fn new( + application_secret: ApplicationSecret, + extra_parameters: &ExtraParameters, + manager: &Manager, + token_path: &Path, + web_channel_handle: &WebEventChannelHandle, + ) -> Self; + fn get_token( + &self, + scopes: &[&str], + ) -> impl Future> + Send + Sync; +} + #[derive(Clone)] -pub(crate) struct Client { +pub(crate) struct BasicClientImpl { basic_client: BasicClient, extra_parameters: ExtraParameters, manager: Manager, @@ -38,8 +53,8 @@ pub(crate) struct Client { web_channel_handle: WebEventChannelHandle, } -impl Client { - pub(crate) fn new( +impl Client for BasicClientImpl { + fn new( application_secret: ApplicationSecret, extra_parameters: &ExtraParameters, manager: &Manager, @@ -58,6 +73,22 @@ impl Client { } } + async fn get_token(&self, scopes: &[&str]) -> Result { + match self.get_token_status_from_file().await { + TokenStatus::Ok(token) => Ok(token.access_token().clone()), + TokenStatus::Expired(refresh_token) => self + .refresh_token(&refresh_token) + .await + .map(|token| token.access_token().clone()), + TokenStatus::Absent => self.retrieve_token(scopes).await.map(|token| { + debug!("Token retrieved: {:?}", token); + token.access_token().clone() + }), + } + } +} + +impl BasicClientImpl { async fn await_code(&self) -> Result { let mut receiver = self.web_channel_handle.get_receiver(); let mut attempts = self.retry_max + 1; @@ -114,20 +145,6 @@ impl Client { result? } - pub(crate) async fn get_token(&self, scopes: &[&str]) -> Result { - match self.get_token_status_from_file().await { - TokenStatus::Ok(token) => Ok(token.access_token().clone()), - TokenStatus::Expired(refresh_token) => self - .refresh_token(&refresh_token) - .await - .map(|token| token.access_token().clone()), - TokenStatus::Absent => self.retrieve_token(scopes).await.map(|token| { - debug!("Token retrieved: {:?}", token); - token.access_token().clone() - }), - } - } - async fn get_token_status_from_file(&self) -> TokenStatus { fs::read_to_string(&self.token_path) .await @@ -303,7 +320,7 @@ mod tests { .token_uri("http://localhost:41047".to_string()) .build() .unwrap(); - let client = Client::new( + let client = BasicClientImpl::new( app_secret, &extra_parameters!("access_type" => "offline"), &get_test_manager(&test_config()), diff --git a/src/domain/oauth2/mod.rs b/src/domain/oauth2/mod.rs index c7bf75e..92ec3f1 100644 --- a/src/domain/oauth2/mod.rs +++ b/src/domain/oauth2/mod.rs @@ -6,6 +6,7 @@ pub(crate) mod extra_parameters; pub(crate) use application_secret::{ApplicationSecret, ApplicationSecretBuilder}; -pub(crate) use client::Client; +pub(crate) use client::BasicClientImpl; +pub use client::Client; pub(crate) use extra_parameters::extra_parameters; diff --git a/src/integration/google/auth/delegate.rs b/src/integration/google/auth/delegate.rs index 924a201..89a4446 100644 --- a/src/integration/google/auth/delegate.rs +++ b/src/integration/google/auth/delegate.rs @@ -5,12 +5,18 @@ use std::future::Future; use std::pin::Pin; #[derive(Builder, Clone)] -pub struct Delegate { - client: Client, +pub struct Delegate +where + ClientImpl: Client, +{ + client: ClientImpl, } -impl Delegate { - async fn get_token( +impl Delegate +where + ClientImpl: Client, +{ + async fn get_secret<'a>( &self, scopes: &[&str], ) -> Result, Box> { @@ -22,7 +28,10 @@ impl Delegate { } } -impl GetToken for Delegate { +impl GetToken for Delegate +where + ClientImpl: Client, +{ fn get_token<'a>( &'a self, scopes: &'a [&str], @@ -33,6 +42,6 @@ impl GetToken for Delegate { + 'a, >, > { - Box::pin(self.get_token(scopes)) + Box::pin(self.get_secret(scopes)) } } diff --git a/src/integration/google/source.rs b/src/integration/google/source.rs index 6b4e8c7..dcbecd6 100644 --- a/src/integration/google/source.rs +++ b/src/integration/google/source.rs @@ -1,6 +1,6 @@ use crate::domain::module_state::NamedModule; use crate::domain::node::{InitReplier, Lifecycle, Manager}; -use crate::domain::oauth2::{extra_parameters, Client}; +use crate::domain::oauth2::{extra_parameters, BasicClientImpl, Client}; use crate::integration::google::auth::web::get_config; use crate::integration::google::auth::DelegateBuilder; use crate::integration::google::tasks::sync; @@ -83,7 +83,7 @@ impl Source { continue; } }; - let client = Client::new( + let client = BasicClientImpl::new( application_secret, &extra_parameters!("access_type" => "offline"), &lifecycle_manager, diff --git a/src/integration/google/tasks/sync.rs b/src/integration/google/tasks/sync.rs index 14818ac..e3cc0a3 100644 --- a/src/integration/google/tasks/sync.rs +++ b/src/integration/google/tasks/sync.rs @@ -1,8 +1,12 @@ +use crate::domain::oauth2::Client; use crate::integration::google::auth::Delegate; use google_tasks1::{hyper_rustls, TasksHub}; use log::info; -pub(crate) async fn sync(delegate: Delegate) { +pub(crate) async fn sync(delegate: Delegate) +where + T: Client, +{ let client = hyper_util::client::legacy::Client::builder(hyper_util::rt::TokioExecutor::new()) .build( hyper_rustls::HttpsConnectorBuilder::new() From c5c798c9df1fccc2b78d8bc9ee0ac8fad6e9dc33 Mon Sep 17 00:00:00 2001 From: Rob Williamson Date: Sat, 4 Jan 2025 09:38:41 +0100 Subject: [PATCH 04/13] Turn CLI Tests into a General Test So Google tests can be added. --- Cargo.toml | 2 +- tests/{cucumber_cli => cucumber}/cli_world.rs | 0 tests/{cucumber_cli => cucumber}/main.rs | 2 +- tests/features/{ => cli}/config.feature | 0 tests/features/{ => cli}/serve.feature | 0 5 files changed, 2 insertions(+), 2 deletions(-) rename tests/{cucumber_cli => cucumber}/cli_world.rs (100%) rename tests/{cucumber_cli => cucumber}/main.rs (82%) rename tests/features/{ => cli}/config.feature (100%) rename tests/features/{ => cli}/serve.feature (100%) diff --git a/Cargo.toml b/Cargo.toml index cd5ab4c..5be8738 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,5 +51,5 @@ panic = "abort" panic = "abort" [[test]] -name = "cucumber_cli" +name = "cucumber" harness = false diff --git a/tests/cucumber_cli/cli_world.rs b/tests/cucumber/cli_world.rs similarity index 100% rename from tests/cucumber_cli/cli_world.rs rename to tests/cucumber/cli_world.rs diff --git a/tests/cucumber_cli/main.rs b/tests/cucumber/main.rs similarity index 82% rename from tests/cucumber_cli/main.rs rename to tests/cucumber/main.rs index 0c8a41d..e6ea59a 100644 --- a/tests/cucumber_cli/main.rs +++ b/tests/cucumber/main.rs @@ -8,6 +8,6 @@ async fn main() { let _ = CliWorld::cucumber() .repeat_skipped() .fail_on_skipped() - .run_and_exit("tests/features") + .run_and_exit("tests/features/cli") .await; } diff --git a/tests/features/config.feature b/tests/features/cli/config.feature similarity index 100% rename from tests/features/config.feature rename to tests/features/cli/config.feature diff --git a/tests/features/serve.feature b/tests/features/cli/serve.feature similarity index 100% rename from tests/features/serve.feature rename to tests/features/cli/serve.feature From 6a8eefeb08969abe4ecc1cd2c561dbd419354a04 Mon Sep 17 00:00:00 2001 From: Rob Williamson Date: Mon, 6 Jan 2025 13:38:09 +0100 Subject: [PATCH 05/13] Add Source Initialisation Wait Test To define the basic Google World struct & execution mechanism. `World` doesn't support explicit lifetimes, so storing `Future`s in the `GoogleWorld` struct isn't an option because they'd all have to live `'static`. --- src/domain/mod.rs | 2 +- src/domain/oauth2/application_secret.rs | 2 +- src/domain/oauth2/extra_parameters.rs | 9 +-- src/integration/google/auth/web.rs | 2 +- src/integration/google/mod.rs | 2 +- src/integration/mod.rs | 2 +- src/lib.rs | 4 +- src/server/mod.rs | 2 +- tests/cucumber/cli_world.rs | 12 +--- tests/cucumber/google_bindings.rs | 20 ++++++ tests/cucumber/google_world.rs | 95 +++++++++++++++++++++++++ tests/cucumber/main.rs | 11 +++ tests/cucumber/shared.rs | 10 +++ tests/features/google/source.feature | 8 +++ 14 files changed, 159 insertions(+), 22 deletions(-) create mode 100644 tests/cucumber/google_bindings.rs create mode 100644 tests/cucumber/google_world.rs create mode 100644 tests/cucumber/shared.rs create mode 100644 tests/features/google/source.feature diff --git a/src/domain/mod.rs b/src/domain/mod.rs index 6f4dfb3..aa84881 100644 --- a/src/domain/mod.rs +++ b/src/domain/mod.rs @@ -4,7 +4,7 @@ pub(crate) mod entity; pub(crate) mod entity_data; pub(crate) mod module_state; pub(crate) mod mpsc_handle; -pub(crate) mod node; +pub mod node; pub(crate) mod oauth2; pub use config::{Config, DomainConfig}; diff --git a/src/domain/oauth2/application_secret.rs b/src/domain/oauth2/application_secret.rs index 1b3c617..409794c 100644 --- a/src/domain/oauth2/application_secret.rs +++ b/src/domain/oauth2/application_secret.rs @@ -5,7 +5,7 @@ use oauth2::{AuthUrl, ClientId, ClientSecret, RedirectUrl, TokenUrl}; use serde::{Deserialize, Serialize}; #[derive(Builder, Deserialize, Getters, Serialize)] -pub(crate) struct ApplicationSecret { +pub struct ApplicationSecret { client_id: String, client_secret: String, auth_uri: String, diff --git a/src/domain/oauth2/extra_parameters.rs b/src/domain/oauth2/extra_parameters.rs index a72d3e6..e8f2cd8 100644 --- a/src/domain/oauth2/extra_parameters.rs +++ b/src/domain/oauth2/extra_parameters.rs @@ -1,6 +1,7 @@ use derive_getters::Getters; use oauth2::{AuthorizationRequest, RefreshTokenRequest}; +#[macro_export] macro_rules! extra_parameters { ($($key: expr => $value: expr),* $(,)?) => { { @@ -14,22 +15,22 @@ macro_rules! extra_parameters { }; } -pub(crate) use extra_parameters; +pub use extra_parameters; #[derive(Clone, Debug, Getters)] -pub(crate) struct ExtraParameter { +pub struct ExtraParameter { key: String, value: String, } impl ExtraParameter { - pub(crate) fn new(key: String, value: String) -> Self { + pub fn new(key: String, value: String) -> Self { assert_allowed_key(&key); Self { key, value } } } -pub(crate) type ExtraParameters = Vec; +pub type ExtraParameters = Vec; macro_rules! apply_extra_parameters { ($self: expr, $extra_parameters: expr) => {{ diff --git a/src/integration/google/auth/web.rs b/src/integration/google/auth/web.rs index ce9593f..0edfb46 100644 --- a/src/integration/google/auth/web.rs +++ b/src/integration/google/auth/web.rs @@ -118,7 +118,7 @@ ConfigQuery, }); impl ConfigQuery { - pub fn to_application_secret(&self, config: &Config) -> ApplicationSecret { + pub(crate) fn to_application_secret(&self, config: &Config) -> ApplicationSecret { ApplicationSecretBuilder::default() .auth_provider_x509_cert_url(Some(self.auth_provider_x509_cert_url())) .auth_uri(self.auth_uri()) diff --git a/src/integration/google/mod.rs b/src/integration/google/mod.rs index 14611be..85509f5 100644 --- a/src/integration/google/mod.rs +++ b/src/integration/google/mod.rs @@ -2,4 +2,4 @@ pub mod auth; mod source; mod tasks; -pub(crate) use source::Source; +pub use source::Source; diff --git a/src/integration/mod.rs b/src/integration/mod.rs index 2de3c21..13cc560 100644 --- a/src/integration/mod.rs +++ b/src/integration/mod.rs @@ -1,3 +1,3 @@ -pub(crate) mod google; +pub mod google; pub(crate) mod log; pub(crate) mod stub; diff --git a/src/lib.rs b/src/lib.rs index dcd027d..8159ed9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,9 +1,9 @@ mod core; pub mod domain; -mod integration; +pub mod integration; mod macros; mod main_impl; -mod server; +pub mod server; mod static_init; mod test; diff --git a/src/server/mod.rs b/src/server/mod.rs index 740c9ad..2b9cc1c 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -11,7 +11,7 @@ mod site_state; mod web_server; mod websocket; -pub(crate) use events::{Code, Event, WebEventChannelHandle}; +pub use events::{Code, Event, WebEventChannelHandle}; #[cfg(test)] pub(crate) use root::format_root_html; diff --git a/tests/cucumber/cli_world.rs b/tests/cucumber/cli_world.rs index bf6470f..2b13e50 100644 --- a/tests/cucumber/cli_world.rs +++ b/tests/cucumber/cli_world.rs @@ -1,4 +1,5 @@ -use cloud_scraper::domain::{Config, DomainConfig}; +use crate::shared::test_config; +use cloud_scraper::domain::Config; use cucumber::gherkin::Step; use cucumber::{given, then, when, World}; use derive_getters::Getters; @@ -339,15 +340,6 @@ async fn a_config_file(_cli_world: &mut CliWorld) { .expect("Error writing config file"); } -fn test_config() -> Config { - Config::with_all_properties( - Some(DomainConfig::new("http://test.domain:8080")), - Some("user@test.domain".to_string()), - None, - None, - ) -} - #[given(regex = r#"an environment variable "([\S ]+)" with the value "([\S ]+)""#)] fn set_environment_variable(cli_world: &mut CliWorld, key: String, value: String) { cli_world.environment_variables.push((key, value)); diff --git a/tests/cucumber/google_bindings.rs b/tests/cucumber/google_bindings.rs new file mode 100644 index 0000000..199e36d --- /dev/null +++ b/tests/cucumber/google_bindings.rs @@ -0,0 +1,20 @@ +use crate::google_world::GoogleWorld; +use crate::shared::test_config; +use cucumber::{given, then, when}; +use std::sync::Arc; + +#[given("a test config")] +fn a_config_file(world: &mut GoogleWorld) { + let config = Arc::new(test_config()); + world.set_config(&config); +} + +#[when("I call run")] +fn i_call_run(world: &mut GoogleWorld) { + world.call_run(); +} + +#[then(regex = r#"it waits.*"#)] +async fn it_waits(world: &mut GoogleWorld) { + assert!(world.run_result().await.timed_out(), "run did not wait"); +} diff --git a/tests/cucumber/google_world.rs b/tests/cucumber/google_world.rs new file mode 100644 index 0000000..38786f5 --- /dev/null +++ b/tests/cucumber/google_world.rs @@ -0,0 +1,95 @@ +use cloud_scraper::domain::node::{LifecycleChannelHandle, Manager}; +use cloud_scraper::domain::Config; +use cloud_scraper::integration::google::Source; +use cloud_scraper::server::WebEventChannelHandle; +use cucumber::World; +use derive_getters::Getters; +use std::sync::{Arc, Once}; +use tokio::sync::{OnceCell, Semaphore}; + +#[derive(Clone, Debug, Getters)] +pub(crate) struct RunResult { + timed_out: bool, +} + +#[derive(Clone, Debug)] +enum Action { + Run, +} + +#[derive(Debug, Getters, World)] +#[world(init = Self::new)] +pub(crate) struct GoogleWorld { + actions: Vec, + execute_once: OnceCell, + manager: Option, + semaphore: Arc, + #[getter(skip)] + source: Option>, + source_once: Once, + web_channel_handle: WebEventChannelHandle, +} + +impl GoogleWorld { + pub(crate) fn new() -> Self { + Self { + actions: Vec::new(), + execute_once: OnceCell::new(), + manager: None, + semaphore: Arc::new(Semaphore::new(1)), + source: None, + source_once: Once::new(), + web_channel_handle: WebEventChannelHandle::new(), + } + } + + pub(crate) fn call_run(&mut self) { + self.actions.push(Action::Run); + } + + pub(crate) async fn run_result(&mut self) -> RunResult { + let actions = self.actions.clone(); + let semaphore = self.semaphore().clone(); + let source = self.source().clone(); + let run_result = self + .execute_once + .get_or_init(|| async move { + let mut run_result = RunResult { timed_out: false }; + for action in actions { + match action { + Action::Run => { + let permit = semaphore.clone().acquire_owned().await.unwrap(); + + run_result.timed_out = tokio::select! { + _ = source.run(permit) => { + false + } + _ = tokio::time::sleep(tokio::time::Duration::from_secs(1)) => { + true + } + }; + } + } + } + + run_result + }) + .await; + + run_result.clone() + } + + pub(crate) fn set_config(&mut self, config: &Arc) { + self.manager = Some(Manager::new(config, LifecycleChannelHandle::new())); + } + + fn source(&mut self) -> Arc { + self.source_once.call_once(|| { + let source = Source::new(&self.manager.as_ref().unwrap(), &self.web_channel_handle); + + self.source.replace(Arc::new(source)); + }); + + self.source.as_mut().unwrap().clone() + } +} diff --git a/tests/cucumber/main.rs b/tests/cucumber/main.rs index e6ea59a..526d8b3 100644 --- a/tests/cucumber/main.rs +++ b/tests/cucumber/main.rs @@ -1,6 +1,12 @@ +extern crate core; + mod cli_world; +mod google_bindings; +mod google_world; +mod shared; use crate::cli_world::CliWorld; +use crate::google_world::GoogleWorld; use cucumber::World; #[tokio::main] @@ -10,4 +16,9 @@ async fn main() { .fail_on_skipped() .run_and_exit("tests/features/cli") .await; + let _ = GoogleWorld::cucumber() + .repeat_skipped() + .fail_on_skipped() + .run_and_exit("tests/features/google") + .await; } diff --git a/tests/cucumber/shared.rs b/tests/cucumber/shared.rs new file mode 100644 index 0000000..2523977 --- /dev/null +++ b/tests/cucumber/shared.rs @@ -0,0 +1,10 @@ +use cloud_scraper::domain::{Config, DomainConfig}; + +pub fn test_config() -> Config { + Config::with_all_properties( + Some(DomainConfig::new("http://test.domain:8080")), + Some("user@test.domain".to_string()), + None, + None, + ) +} diff --git a/tests/features/google/source.feature b/tests/features/google/source.feature new file mode 100644 index 0000000..78985ad --- /dev/null +++ b/tests/features/google/source.feature @@ -0,0 +1,8 @@ +# Created by robwilliamson at 1/4/25 +Feature: Google Source + Google data source. + + Scenario: Before Initialisation + Given a test config + When I call run + Then it waits for initialisation From 9d310b7fbf5d81b75aa21a742f074a9faa7cf92c Mon Sep 17 00:00:00 2001 From: Rob Williamson Date: Mon, 6 Jan 2025 14:09:17 +0100 Subject: [PATCH 06/13] Check Init Causes Confirmation And that the semaphore permit is released on initialisation. --- src/domain/mod.rs | 1 + tests/cucumber/google_bindings.rs | 21 +++++++++ tests/cucumber/google_world.rs | 70 +++++++++++++++++++++------- tests/features/google/source.feature | 9 +++- 4 files changed, 84 insertions(+), 17 deletions(-) diff --git a/src/domain/mod.rs b/src/domain/mod.rs index aa84881..bd01185 100644 --- a/src/domain/mod.rs +++ b/src/domain/mod.rs @@ -8,3 +8,4 @@ pub mod node; pub(crate) mod oauth2; pub use config::{Config, DomainConfig}; +pub use mpsc_handle::one_shot; diff --git a/tests/cucumber/google_bindings.rs b/tests/cucumber/google_bindings.rs index 199e36d..b3a26ca 100644 --- a/tests/cucumber/google_bindings.rs +++ b/tests/cucumber/google_bindings.rs @@ -14,7 +14,28 @@ fn i_call_run(world: &mut GoogleWorld) { world.call_run(); } +#[when("I send_init")] +fn i_send_init(world: &mut GoogleWorld) { + world.send_init(); +} + +#[then("it releases the semaphore")] +async fn it_releases_the_semaphore(world: &mut GoogleWorld) { + assert!( + world.run_result().await.semaphore_released(), + "run did not release semaphore" + ); +} + #[then(regex = r#"it waits.*"#)] async fn it_waits(world: &mut GoogleWorld) { assert!(world.run_result().await.timed_out(), "run did not wait"); } + +#[then("it replies to init with ()")] +async fn it_replies_to_init_with(world: &mut GoogleWorld) { + assert!( + world.run_result().await.replied_to_init(), + "run did not reply to init with ()" + ); +} diff --git a/tests/cucumber/google_world.rs b/tests/cucumber/google_world.rs index 38786f5..b29ff73 100644 --- a/tests/cucumber/google_world.rs +++ b/tests/cucumber/google_world.rs @@ -1,5 +1,5 @@ use cloud_scraper::domain::node::{LifecycleChannelHandle, Manager}; -use cloud_scraper::domain::Config; +use cloud_scraper::domain::{one_shot, Config}; use cloud_scraper::integration::google::Source; use cloud_scraper::server::WebEventChannelHandle; use cucumber::World; @@ -9,11 +9,24 @@ use tokio::sync::{OnceCell, Semaphore}; #[derive(Clone, Debug, Getters)] pub(crate) struct RunResult { + replied_to_init: bool, + semaphore_released: bool, timed_out: bool, } +impl RunResult { + fn new() -> Self { + Self { + replied_to_init: false, + semaphore_released: false, + timed_out: false, + } + } +} + #[derive(Clone, Debug)] enum Action { + Init, Run, } @@ -22,6 +35,7 @@ enum Action { pub(crate) struct GoogleWorld { actions: Vec, execute_once: OnceCell, + #[getter(skip)] manager: Option, semaphore: Arc, #[getter(skip)] @@ -31,32 +45,37 @@ pub(crate) struct GoogleWorld { } impl GoogleWorld { - pub(crate) fn new() -> Self { - Self { - actions: Vec::new(), - execute_once: OnceCell::new(), - manager: None, - semaphore: Arc::new(Semaphore::new(1)), - source: None, - source_once: Once::new(), - web_channel_handle: WebEventChannelHandle::new(), - } - } - pub(crate) fn call_run(&mut self) { self.actions.push(Action::Run); } + pub(crate) fn send_init(&mut self) { + self.actions.push(Action::Init); + } + + pub(crate) fn set_config(&mut self, config: &Arc) { + self.manager = Some(Manager::new(config, LifecycleChannelHandle::new())); + } +} + +impl GoogleWorld { pub(crate) async fn run_result(&mut self) -> RunResult { let actions = self.actions.clone(); let semaphore = self.semaphore().clone(); let source = self.source().clone(); + let mut manager = self.manager().clone(); let run_result = self .execute_once .get_or_init(|| async move { - let mut run_result = RunResult { timed_out: false }; + let mut run_result = RunResult::new(); for action in actions { match action { + Action::Init => { + let (sender, mut receiver) = one_shot::<()>(); + manager.send_init(sender).unwrap(); + receiver.recv().await.unwrap(); + run_result.replied_to_init = true; + } Action::Run => { let permit = semaphore.clone().acquire_owned().await.unwrap(); @@ -68,6 +87,8 @@ impl GoogleWorld { true } }; + + run_result.semaphore_released = semaphore.available_permits() == 1; } } } @@ -76,11 +97,28 @@ impl GoogleWorld { }) .await; + // Send a stop in case there are any tasks still running. + let _ = self.manager().clone().send_stop(); + run_result.clone() } +} - pub(crate) fn set_config(&mut self, config: &Arc) { - self.manager = Some(Manager::new(config, LifecycleChannelHandle::new())); +impl GoogleWorld { + pub(crate) fn new() -> Self { + Self { + actions: Vec::new(), + execute_once: OnceCell::new(), + manager: None, + semaphore: Arc::new(Semaphore::new(1)), + source: None, + source_once: Once::new(), + web_channel_handle: WebEventChannelHandle::new(), + } + } + + fn manager(&self) -> &Manager { + self.manager.as_ref().unwrap() } fn source(&mut self) -> Arc { diff --git a/tests/features/google/source.feature b/tests/features/google/source.feature index 78985ad..8def981 100644 --- a/tests/features/google/source.feature +++ b/tests/features/google/source.feature @@ -5,4 +5,11 @@ Feature: Google Source Scenario: Before Initialisation Given a test config When I call run - Then it waits for initialisation + Then it releases the semaphore + And it waits for initialisation + + Scenario: After Initialisation + Given a test config + When I call run + When I send_init + Then it replies to init with () From 1b7eb202d0fed738a7cecfd303dc2bcae60d5815 Mon Sep 17 00:00:00 2001 From: Rob Williamson Date: Mon, 6 Jan 2025 18:31:42 +0100 Subject: [PATCH 07/13] Test Stopping Early And make `source.run` execute always - the run action just waits until the `source.run`'s future has released a semaphore permit. The assumption is that this means the source has started running, and other signals can be sent. Only do any of this once. --- tests/cucumber/google_bindings.rs | 10 +++ tests/cucumber/google_world.rs | 113 +++++++++++++++++++-------- tests/features/google/source.feature | 7 ++ 3 files changed, 98 insertions(+), 32 deletions(-) diff --git a/tests/cucumber/google_bindings.rs b/tests/cucumber/google_bindings.rs index b3a26ca..4f10011 100644 --- a/tests/cucumber/google_bindings.rs +++ b/tests/cucumber/google_bindings.rs @@ -19,6 +19,11 @@ fn i_send_init(world: &mut GoogleWorld) { world.send_init(); } +#[when("I send_stop")] +fn i_send_stop(world: &mut GoogleWorld) { + world.send_stop(); +} + #[then("it releases the semaphore")] async fn it_releases_the_semaphore(world: &mut GoogleWorld) { assert!( @@ -32,6 +37,11 @@ async fn it_waits(world: &mut GoogleWorld) { assert!(world.run_result().await.timed_out(), "run did not wait"); } +#[then(regex = r#"it does not wait.*"#)] +async fn it_does_not_wait(world: &mut GoogleWorld) { + assert!(!world.run_result().await.timed_out(), "run waited"); +} + #[then("it replies to init with ()")] async fn it_replies_to_init_with(world: &mut GoogleWorld) { assert!( diff --git a/tests/cucumber/google_world.rs b/tests/cucumber/google_world.rs index b29ff73..4f1406a 100644 --- a/tests/cucumber/google_world.rs +++ b/tests/cucumber/google_world.rs @@ -4,8 +4,11 @@ use cloud_scraper::integration::google::Source; use cloud_scraper::server::WebEventChannelHandle; use cucumber::World; use derive_getters::Getters; +use std::fmt::Debug; +use std::future::Future; use std::sync::{Arc, Once}; -use tokio::sync::{OnceCell, Semaphore}; +use tokio::sync::{OnceCell, OwnedSemaphorePermit, Semaphore}; +use tokio::{join, select}; #[derive(Clone, Debug, Getters)] pub(crate) struct RunResult { @@ -24,10 +27,11 @@ impl RunResult { } } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq)] enum Action { Init, Run, + Stop, } #[derive(Debug, Getters, World)] @@ -37,7 +41,6 @@ pub(crate) struct GoogleWorld { execute_once: OnceCell, #[getter(skip)] manager: Option, - semaphore: Arc, #[getter(skip)] source: Option>, source_once: Once, @@ -53,6 +56,10 @@ impl GoogleWorld { self.actions.push(Action::Init); } + pub(crate) fn send_stop(&mut self) { + self.actions.push(Action::Stop); + } + pub(crate) fn set_config(&mut self, config: &Arc) { self.manager = Some(Manager::new(config, LifecycleChannelHandle::new())); } @@ -60,47 +67,90 @@ impl GoogleWorld { impl GoogleWorld { pub(crate) async fn run_result(&mut self) -> RunResult { + if self.execute_once.initialized() { + return self.execute_once.get().unwrap().clone(); + } + + async fn test<'a>( + permit: OwnedSemaphorePermit, + semaphore: &'a Arc, + source: &'a Arc, + ) { + drop(permit); + source + .run(semaphore.clone().acquire_owned().await.unwrap()) + .await; + } + + async fn wait_for_timeout<'a, T>(future: T) -> bool + where + T: Future + 'a, + { + select! { + _ = future => { + false + } + _ = tokio::time::sleep(tokio::time::Duration::from_secs(1)) => { + true + } + } + } + + async fn runner<'a>( + actions: &Vec, + manager: &mut Manager, + test_semaphore: &Arc, + ) -> RunResult { + let mut run_result = RunResult::new(); + + for action in actions { + match action { + Action::Init => { + let (sender, mut receiver) = one_shot::<()>(); + manager.send_init(sender).unwrap(); + receiver.recv().await.unwrap(); + run_result.replied_to_init = true; + } + Action::Run => { + // Wait until the test is actually executing. + let _ = test_semaphore.acquire().await.unwrap(); + } + Action::Stop => { + manager.send_stop().unwrap(); + } + } + } + + run_result + } + let actions = self.actions.clone(); - let semaphore = self.semaphore().clone(); + let semaphore = Arc::new(Semaphore::new(1)); + let test_semaphore = Arc::new(Semaphore::new(1)); + let test_permit = test_semaphore.clone().acquire_owned().await.unwrap(); let source = self.source().clone(); let mut manager = self.manager().clone(); + let run_result = self .execute_once .get_or_init(|| async move { - let mut run_result = RunResult::new(); - for action in actions { - match action { - Action::Init => { - let (sender, mut receiver) = one_shot::<()>(); - manager.send_init(sender).unwrap(); - receiver.recv().await.unwrap(); - run_result.replied_to_init = true; - } - Action::Run => { - let permit = semaphore.clone().acquire_owned().await.unwrap(); - - run_result.timed_out = tokio::select! { - _ = source.run(permit) => { - false - } - _ = tokio::time::sleep(tokio::time::Duration::from_secs(1)) => { - true - } - }; - - run_result.semaphore_released = semaphore.available_permits() == 1; - } - } - } + let test = Box::pin(test(test_permit, &semaphore, &source)); + let runner = runner(&actions, &mut manager, &test_semaphore); + let race = wait_for_timeout(test); + let (mut run_result, timed_out) = join!(runner, race); + + run_result.timed_out = timed_out; + run_result.semaphore_released = semaphore.available_permits() == 1; run_result }) - .await; + .await + .clone(); // Send a stop in case there are any tasks still running. let _ = self.manager().clone().send_stop(); - run_result.clone() + run_result } } @@ -110,7 +160,6 @@ impl GoogleWorld { actions: Vec::new(), execute_once: OnceCell::new(), manager: None, - semaphore: Arc::new(Semaphore::new(1)), source: None, source_once: Once::new(), web_channel_handle: WebEventChannelHandle::new(), diff --git a/tests/features/google/source.feature b/tests/features/google/source.feature index 8def981..2968ddd 100644 --- a/tests/features/google/source.feature +++ b/tests/features/google/source.feature @@ -8,6 +8,13 @@ Feature: Google Source Then it releases the semaphore And it waits for initialisation + Scenario: Stop Before Initialisation + Given a test config + When I call run + When I send_stop + Then it releases the semaphore + And it does not wait for initialisation + Scenario: After Initialisation Given a test config When I call run From eaabe7021e7ae9c4f1227c5f014e11dfb1e6ed85 Mon Sep 17 00:00:00 2001 From: Rob Williamson Date: Wed, 8 Jan 2025 22:45:11 +0100 Subject: [PATCH 08/13] Make a Test Client Incomplete because it needs to be able to mock things. --- src/core/engine.rs | 4 +- src/core/module.rs | 6 +- src/domain/mod.rs | 3 +- src/domain/oauth2/client.rs | 12 +- src/domain/oauth2/config.rs | 132 ++++++++++++++++ src/domain/oauth2/mod.rs | 6 +- src/integration/google/auth/config.rs | 122 +++++++++++++++ src/integration/google/auth/delegate.rs | 2 +- src/integration/google/auth/mod.rs | 2 + src/integration/google/auth/web.rs | 191 +++--------------------- src/integration/google/source.rs | 44 ++++-- src/lib.rs | 2 +- src/server/errors.rs | 36 ++++- src/static_init/error.rs | 18 ++- tests/cucumber/google/client.rs | 63 ++++++++ tests/cucumber/google/mod.rs | 1 + tests/cucumber/google_world.rs | 7 +- tests/cucumber/main.rs | 1 + tests/features/cli/serve.feature | 5 +- 19 files changed, 451 insertions(+), 206 deletions(-) create mode 100644 src/domain/oauth2/config.rs create mode 100644 src/integration/google/auth/config.rs create mode 100644 tests/cucumber/google/client.rs create mode 100644 tests/cucumber/google/mod.rs diff --git a/src/core/engine.rs b/src/core/engine.rs index 06b2ddf..6bc8938 100644 --- a/src/core/engine.rs +++ b/src/core/engine.rs @@ -9,6 +9,7 @@ use crate::core::node_handles::NodeHandles; use crate::domain::config::Config; use crate::domain::mpsc_handle::{one_shot, OneshotMpscSenderHandle}; use crate::domain::node::{LifecycleChannelHandle, Manager}; +use crate::domain::oauth2::BasicClientImpl; use crate::integration::google::Source as GoogleSource; use crate::integration::log::Sink as LogSink; use crate::integration::stub::Source as StubSource; @@ -66,7 +67,8 @@ where let wait_duration = self.manager.core_config().exit_after(); let mut stub_source = StubSource::new(&self.manager); - let google_source = GoogleSource::new(&self.manager, self.server.get_web_channel_handle()); + let google_source: GoogleSource = + GoogleSource::new(&self.manager, self.server.get_web_channel_handle()); let mut log_sink = LogSink::new(&self.manager, &stub_source.get_readonly_channel_handle()); let node_handles = NodeHandles::new(&self.manager, self.server.get_web_channel_handle()); diff --git a/src/core/module.rs b/src/core/module.rs index c3cc555..33863df 100644 --- a/src/core/module.rs +++ b/src/core/module.rs @@ -13,11 +13,15 @@ impl ModuleState for State { #[cfg(test)] mod tests { use super::*; + use crate::domain::oauth2::BasicClientImpl; use crate::integration::google::Source; #[tokio::test] async fn test_path_for() { assert_eq!(State::path(), "state"); - assert_eq!(State::path_for::().await.unwrap(), "state/google") + assert_eq!( + State::path_for::>().await.unwrap(), + "state/google" + ) } } diff --git a/src/domain/mod.rs b/src/domain/mod.rs index bd01185..868e35f 100644 --- a/src/domain/mod.rs +++ b/src/domain/mod.rs @@ -5,7 +5,6 @@ pub(crate) mod entity_data; pub(crate) mod module_state; pub(crate) mod mpsc_handle; pub mod node; -pub(crate) mod oauth2; - +pub mod oauth2; pub use config::{Config, DomainConfig}; pub use mpsc_handle::one_shot; diff --git a/src/domain/oauth2/client.rs b/src/domain/oauth2/client.rs index efbf133..8f036cd 100644 --- a/src/domain/oauth2/client.rs +++ b/src/domain/oauth2/client.rs @@ -297,7 +297,7 @@ impl BasicClientImpl { } #[cfg(test)] -mod tests { +pub mod tests { use super::*; mod make_redirect_url { @@ -379,4 +379,14 @@ mod tests { ); } } + + mod access_token { + use oauth2::AccessToken; + + #[test] + fn test_is_send_and_sync() { + fn is_send_and_sync() {} + is_send_and_sync::(); + } + } } diff --git a/src/domain/oauth2/config.rs b/src/domain/oauth2/config.rs new file mode 100644 index 0000000..af94b78 --- /dev/null +++ b/src/domain/oauth2/config.rs @@ -0,0 +1,132 @@ +use paste::paste; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::fmt::Debug; +use std::future::Future; +use std::path::Path; + +macro_rules! make_config { + ($the_trait:ident, { $($e:ident),* }) => { + paste! { + pub trait $the_trait: Debug + Deserialize<'static> + From<&'static HashMap> + Serialize { + $( + fn $e(&self) -> &str; + )* + } + } + }; +} + +macro_rules! make_config_struct { + ($struct:ident, $the_trait:ident, { $($e:ident),* }, { $($d:ident, $v:literal),* }) => { + paste! { + #[derive(Builder, Debug, Deserialize, Serialize)] + pub struct $struct { + $( + $e: String, + )* + $( + $d: String, + )* + } + + impl $the_trait for $struct { + $( + fn $e(&self) -> &str { + &self.$e + } + )* + + $( + fn $d(&self) -> &str { + &self.$d + } + )* + } + + impl From<&HashMap> for $struct { + fn from(map: &HashMap) -> Self { + Self { + $( + $e: map.get(stringify!($e)).expect(&Self::format_missing_hash_key_message(stringify!($e), map)).clone(), + )* + $( + $d: map.get(stringify!($d)).unwrap_or(&String::from($v)).clone(), + )* + } + } + } + + impl $struct { + fn format_missing_hash_key_message(key: &str, map: &HashMap) -> String { + format!("Could not get required value {} from {:?}", key, map) + } + } + } + }; +} + +pub(crate) use make_config_struct; + +make_config!( + Config, + { + auth_uri, + auth_provider_x509_cert_url, + client_id, + client_secret, + project_id, + token_uri + } +); + +pub trait PersistableConfig: Config { + fn persist( + &self, + path: &Path, + ) -> impl Future> + Send + Sync; + fn read_config(path: &Path) + -> impl Future> + Send + Sync; +} + +#[cfg(test)] +mod test { + use super::*; + use derive_builder::Builder; + + make_config!( + TestConfig, + { a, b } + ); + + make_config_struct!( + TestConfigStruct, TestConfig, { a }, { b, "c" } + ); + + #[test] + fn constructor_works() { + let config: TestConfigStruct = TestConfigStructBuilder::default() + .a("a".into()) + .b("b".into()) + .build() + .expect("Failed to build TestConfigStruct"); + assert_eq!(config.a(), "a"); + assert_eq!(config.b(), "b"); + } + + #[test] + fn from_hash_map_works() { + let mut map = HashMap::new(); + map.insert("a".into(), "a".into()); + let config: TestConfigStruct = (&map).into(); + assert_eq!(config.a(), "a"); + assert_eq!(config.b(), "c"); + } + + #[test] + #[should_panic] + fn from_hash_map_missing_key() { + let map = HashMap::new(); + let _config: TestConfigStruct = (&map).into(); + } +} diff --git a/src/domain/oauth2/mod.rs b/src/domain/oauth2/mod.rs index 92ec3f1..4dd9983 100644 --- a/src/domain/oauth2/mod.rs +++ b/src/domain/oauth2/mod.rs @@ -2,11 +2,13 @@ mod application_secret; mod client; mod token; +mod config; pub(crate) mod extra_parameters; -pub(crate) use application_secret::{ApplicationSecret, ApplicationSecretBuilder}; +pub use application_secret::{ApplicationSecret, ApplicationSecretBuilder}; pub(crate) use client::BasicClientImpl; pub use client::Client; -pub(crate) use extra_parameters::extra_parameters; +pub(crate) use config::{make_config_struct, Config, PersistableConfig}; +pub use extra_parameters::{extra_parameters, ExtraParameter, ExtraParameters}; diff --git a/src/integration/google/auth/config.rs b/src/integration/google/auth/config.rs new file mode 100644 index 0000000..852b73f --- /dev/null +++ b/src/integration/google/auth/config.rs @@ -0,0 +1,122 @@ +use crate::domain::oauth2::Config as Oauth2Config; +use crate::domain::oauth2::{ + make_config_struct, ApplicationSecret, ApplicationSecretBuilder, PersistableConfig, +}; +use crate::domain::Config; +use derive_builder::Builder; +use log::debug; +use paste::paste; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::io; +use std::path::Path; +use tokio::fs; + +macro_rules! make_config_query { + ($struct:ident, { $($e:ident),* }, { $($d:ident, $v:literal),* }) => { + make_config_struct!( + $struct, + Oauth2Config, + { $($e),* }, + { $($d, $v),* } + ); + paste! { + impl $struct { + pub(crate) fn empty_page_data() -> HashMap<&'static str, String> { + let mut page_data = HashMap::new(); + $( + page_data.insert(stringify!($e), Self::format_empty(stringify!($e))); + )* + $( + page_data.insert(stringify!($d), Self::format(stringify!($d), $v)); + )* + page_data + } + + fn format(name: &str, value: &str) -> String { + format!("name=\"{}\" value=\"{}\"", name, value) + } + + fn format_empty(name: &str) -> String { + format!("name=\"{}\"", name) + } + + pub(crate) fn to_page_data(&self) -> HashMap<&'static str, String> { + let mut page_data = HashMap::new(); + $( + page_data.insert(stringify!($e), Self::format(stringify!($e), &self.$e)); + )* + $( + page_data.insert(stringify!($d), Self::format(stringify!($d), &self.$d)); + )* + page_data + } + } + } + } +} +make_config_query!( + ConfigQuery, + { project_id, client_id, client_secret }, + { + auth_uri, "https://accounts.google.com/o/oauth2/auth", + auth_provider_x509_cert_url, "https://www.googleapis.com/oauth2/v1/certs", + token_uri, "https://oauth2.googleapis.com/token" + } +); + +impl PersistableConfig for ConfigQuery { + async fn persist(&self, path: &Path) -> Result<(), io::Error> { + debug!("Config path: {:?}", path); + let serialized = serde_yaml::to_string(self).map_err(|e| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("Could not serialize config file due to {:?}.", e), + ) + })?; + + fs::write(&path, serialized).await?; + + Ok(()) + } + + async fn read_config(path: &Path) -> Result { + debug!("Config path: {:?}", path); + let slice = fs::read(&path).await.map_err(|e| { + debug!( + "Could not read config file at {} due to {:?}.", + path.display(), + e + ); + e + })?; + debug!("Read result: {:?}", slice); + let config_query = serde_yaml::from_slice(&slice).map_err(|e| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("Could not parse config file due to {:?}.", e), + ) + })?; + debug!("Parse result: {:?}", config_query); + Ok(config_query) + } +} + +impl ConfigQuery { + pub(crate) fn to_application_secret(&self, config: &Config) -> ApplicationSecret { + ApplicationSecretBuilder::default() + .auth_provider_x509_cert_url(Some(self.auth_provider_x509_cert_url().into())) + .auth_uri(self.auth_uri().into()) + .client_email(None) + .client_id(self.client_id().into()) + .client_secret(self.client_secret().into()) + .client_x509_cert_url(None) + .project_id(Some(self.project_id().into())) + .redirect_uris(vec![config.redirect_uri()]) + .token_uri(self.token_uri().into()) + .build() + .unwrap_or_else(|e| { + panic!("Error while building ApplicationSecret: {:?}", e); + }) + } +} diff --git a/src/integration/google/auth/delegate.rs b/src/integration/google/auth/delegate.rs index 89a4446..4274b57 100644 --- a/src/integration/google/auth/delegate.rs +++ b/src/integration/google/auth/delegate.rs @@ -16,7 +16,7 @@ impl Delegate where ClientImpl: Client, { - async fn get_secret<'a>( + async fn get_secret( &self, scopes: &[&str], ) -> Result, Box> { diff --git a/src/integration/google/auth/mod.rs b/src/integration/google/auth/mod.rs index 881e976..b13448f 100644 --- a/src/integration/google/auth/mod.rs +++ b/src/integration/google/auth/mod.rs @@ -1,5 +1,7 @@ +mod config; mod delegate; pub mod web; +pub(crate) use config::ConfigQuery; pub use delegate::Delegate; pub use delegate::DelegateBuilder; diff --git a/src/integration/google/auth/web.rs b/src/integration/google/auth/web.rs index 0edfb46..0194927 100644 --- a/src/integration/google/auth/web.rs +++ b/src/integration/google/auth/web.rs @@ -1,24 +1,16 @@ -use crate::core::module::State; use crate::core::node_handles::NodeHandles; -use crate::domain::config::Config; -use crate::domain::module_state::ModuleState; use crate::domain::node::Manager; -use crate::domain::oauth2::ApplicationSecret; -use crate::domain::oauth2::ApplicationSecretBuilder; +use crate::domain::oauth2::BasicClientImpl; +use crate::domain::oauth2::PersistableConfig; +use crate::integration::google::auth::ConfigQuery; use crate::integration::google::Source; use crate::server::auth::auth_validation; use crate::server::errors::Rejectable; use crate::server::javascript::WithRedirect; -use crate::static_init::error::{Error, IoErrorExt, SerdeErrorExt}; use handlebars::Handlebars; use lazy_static::lazy_static; use log::{debug, error}; -use paste::paste; -use serde::{Deserialize, Serialize}; use std::collections::HashMap; -use std::io; -use std::path::PathBuf; -use tokio::fs; use warp::{path, reply, Filter, Rejection, Reply}; const CONFIG_TEMPLATE: &str = "config/google"; @@ -37,105 +29,6 @@ lazy_static! { }; } -macro_rules! make_config_query { - ($struct:ident, { $($e:ident),* }, { $($d:ident, $v:literal),* }) => { - paste! { - #[derive(Debug, Deserialize, Serialize)] - pub struct $struct { - $( - $e: String, - )* - $( - $d: String, - )* - } - - impl $struct { - fn empty_page_data() -> HashMap<&'static str, String> { - let mut page_data = HashMap::new(); - $( - page_data.insert(stringify!($e), Self::format_empty(stringify!($e))); - )* - $( - page_data.insert(stringify!($d), Self::format(stringify!($d), $v)); - )* - page_data - } - - pub fn new(map: &HashMap) -> Self { - Self { - $( - $e: map.get(stringify!($e)).unwrap_or(&String::new()) - .clone(), - )* - $( - $d: map.get(stringify!($d)).unwrap_or - (&String::from($v)).clone(), - )* - } - } - - fn format(name: &str, value: &str) -> String { - format!("name=\"{}\" value=\"{}\"", name, value) - } - - fn format_empty(name: &str) -> String { - format!("name=\"{}\"", name) - } - - fn to_page_data(&self) -> HashMap<&'static str, String> { - let mut page_data = HashMap::new(); - $( - page_data.insert(stringify!($e), Self::format(stringify!($e), &self.$e)); - )* - $( - page_data.insert(stringify!($d), Self::format(stringify!($d), &self.$d)); - )* - page_data - } - - $( - pub fn $e(&self) -> String { - self.$e.clone() - } - )* - $( - pub fn $d(&self) -> String { - self.$d.clone() - } - )* - } - } - } -} -make_config_query!( -ConfigQuery, -{ project_id, client_id, client_secret }, -{ - auth_uri, "https://accounts.google.com/o/oauth2/auth", - auth_provider_x509_cert_url, "https://www.googleapis.com/oauth2/v1/certs", - token_uri, "https://oauth2.googleapis.com/token" -}); - -impl ConfigQuery { - pub(crate) fn to_application_secret(&self, config: &Config) -> ApplicationSecret { - ApplicationSecretBuilder::default() - .auth_provider_x509_cert_url(Some(self.auth_provider_x509_cert_url())) - .auth_uri(self.auth_uri()) - .client_email(None) - .client_id(self.client_id()) - .client_secret(self.client_secret()) - .client_x509_cert_url(None) - .project_id(Some(self.project_id())) - .redirect_uris(vec![config.redirect_uri()]) - .token_uri(self.token_uri()) - .build() - .unwrap_or_else(|e| { - panic!("Error while building ApplicationSecret: {:?}", e); - }) - } -} - pub fn config_google( handles: &NodeHandles, ) -> impl Filter + Clone { @@ -165,7 +58,7 @@ pub fn config_google( } async fn format_response(handles: NodeHandles) -> Result { - let existing_config = get_config().await; + let existing_config = Source::::get_auth_config().await.ok(); Ok(reply::html( format_config_google_html(handles, &existing_config).await, )) @@ -189,12 +82,15 @@ async fn update_config( form_map: HashMap, handles: NodeHandles, ) -> Result { - let config = ConfigQuery::new(&form_map); + let config = ConfigQuery::from(&form_map); - match put_config(&config).await { + let path = Source::::config_path() + .await + .map_err(|e| e.into_rejection())?; + match config.persist(&path).await { Ok(_) => { let mut sender: Manager = handles.lifecycle_manager().clone(); - match sender.send_read_config::() { + match sender.send_read_config::>() { Ok(_) => { debug!("Google config update sent"); Ok(warp::redirect::found(warp::http::Uri::from_static( @@ -211,55 +107,15 @@ async fn update_config( } } -async fn config_path() -> Result { - let root = State::path_for::().await?; - debug!("Root: {:?}", root); - Ok(PathBuf::from(root).join("config.yaml")) -} - -async fn put_config(config_query: &ConfigQuery) -> Result<(), Error> { - let config_path = config_path() - .await - .map_err(|e| e.to_source_creation_builder_error())?; - - debug!("Config path: {:?}", config_path); - - let serialized = - serde_yaml::to_string(config_query).map_err(|e| e.to_yaml_serialization_error())?; - - fs::write(&config_path, serialized) - .await - .map_err(|e| e.to_source_creation_builder_error())?; - - Ok(()) -} - -pub async fn get_config() -> Option { - let config_path = config_path().await; - - if let Ok(config_path) = config_path { - debug!("Config path: {:?}", config_path); - let read_result = fs::read(&config_path).await; - debug!("Read result: {:?}", read_result); - if let Ok(config) = read_result { - let parse_result = serde_yaml::from_slice(&config); - debug!("Parse result: {:?}", parse_result); - if let Ok(config) = parse_result { - return Some(config); - } - } - } - - None -} - #[cfg(test)] mod tests { use super::*; + use crate::integration::google::auth::config::ConfigQueryBuilder; use crate::server::auth::gen_token_for_path; use crate::test::tests::CleanableTestFile; use lazy_static::lazy_static; use std::sync::Mutex; + use tokio::fs; use warp::http::header::COOKIE; use warp::http::StatusCode; use warp::test::request; @@ -271,7 +127,7 @@ mod tests { async fn make_config_file_and_lock<'a>() -> CleanableTestFile<'a> { CleanableTestFile::new( TEST_MUTEX.lock().expect("Could not lock mutex."), - config_path() + Source::::config_path() .await .expect("Could not get config path.") .to_str() @@ -287,19 +143,20 @@ mod tests { } async fn reset() { - let config_path = config_path().await.unwrap(); + let config_path = Source::::config_path().await.unwrap(); let _ = fs::remove_file(&config_path).await; } fn test_config() -> ConfigQuery { - ConfigQuery { - project_id: "test_project_id".to_string(), - client_id: "test_client_id".to_string(), - client_secret: "test_client_secret".to_string(), - auth_uri: "https://test.auth.uri".to_string(), - auth_provider_x509_cert_url: "test_auth_provider_x509_cert_url".to_string(), - token_uri: "https://test.token.uri".to_string(), - } + ConfigQueryBuilder::default() + .project_id("test_project_id".into()) + .client_id("test_client_id".into()) + .client_secret("test_client_secret".into()) + .auth_uri("https://test.auth.uri".into()) + .auth_provider_x509_cert_url("test_auth_provider_x509_cert_url".into()) + .token_uri("https://test.token.uri".into()) + .build() + .expect("Could not build test config") } fn test_config_form_encoded() -> String { @@ -421,7 +278,7 @@ mod tests { mod to_application_secret { use super::*; - use crate::domain::DomainConfig; + use crate::domain::{Config, DomainConfig}; #[test] fn returns_application_secret() { diff --git a/src/integration/google/source.rs b/src/integration/google/source.rs index dcbecd6..552b5ec 100644 --- a/src/integration/google/source.rs +++ b/src/integration/google/source.rs @@ -1,14 +1,17 @@ -use crate::domain::module_state::NamedModule; +use crate::core::module::State; +use crate::domain::module_state::{ModuleState, NamedModule}; use crate::domain::node::{InitReplier, Lifecycle, Manager}; -use crate::domain::oauth2::{extra_parameters, BasicClientImpl, Client}; -use crate::integration::google::auth::web::get_config; -use crate::integration::google::auth::DelegateBuilder; +use crate::domain::oauth2::{extra_parameters, BasicClientImpl, Client, PersistableConfig}; +use crate::integration::google::auth::{ConfigQuery, DelegateBuilder}; use crate::integration::google::tasks::sync; use crate::server::auth::get_token_path; use crate::server::WebEventChannelHandle; use derive_getters::Getters; -use log::{error, info, trace}; +use log::{debug, error, info, trace}; use std::any::TypeId; +use std::io; +use std::marker::PhantomData; +use std::path::PathBuf; use std::sync::Arc; use std::time::Duration; use tokio::sync::{mpsc, OwnedSemaphorePermit, Semaphore}; @@ -17,20 +20,31 @@ use tokio::{join, task}; use Lifecycle::{Init, ReadConfig, Stop}; #[derive(Clone, Debug, Getters)] -pub struct Source { +pub struct Source +where + ClientType: Client, +{ + _phantom: PhantomData, lifecycle_manager: Manager, web_channel_handle: WebEventChannelHandle, } -impl NamedModule for Source { +impl NamedModule for Source +where + ClientType: Client, +{ fn name() -> &'static str { "google" } } -impl Source { +impl Source +where + ClientType: Client, +{ pub fn new(manager: &Manager, web_channel_handle: &WebEventChannelHandle) -> Self { Self { + _phantom: Default::default(), lifecycle_manager: manager.clone(), web_channel_handle: web_channel_handle.clone(), } @@ -70,7 +84,7 @@ impl Source { } } - let application_secret = if let Some(config) = get_config().await { + let application_secret = if let Ok(config) = Self::get_auth_config().await { config.to_application_secret(&core_config) } else { Self::wait_in_loop().await; @@ -151,7 +165,7 @@ impl Source { event.reply_to_init_with((), "google_source").await } ReadConfig(type_id) => { - if type_id == TypeId::of::() { + if type_id == TypeId::of::>() { send_load!(); } } @@ -175,6 +189,16 @@ impl Source { let (_task_result, _stop_result) = join!(task, lifetime_task); } + pub(crate) async fn config_path() -> Result { + let root = State::path_for::().await?; + debug!("Root: {:?}", root); + Ok(PathBuf::from(root).join("config.yaml")) + } + + pub(crate) async fn get_auth_config() -> Result { + ConfigQuery::read_config(&Self::config_path().await?).await + } + async fn wait_in_loop() { sleep(Duration::from_secs(10)).await; } diff --git a/src/lib.rs b/src/lib.rs index 8159ed9..7c7cc38 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,7 +4,7 @@ pub mod integration; mod macros; mod main_impl; pub mod server; -mod static_init; +pub mod static_init; mod test; pub use main_impl::{main_impl, CoreInterface}; diff --git a/src/server/errors.rs b/src/server/errors.rs index 52d7d3e..96b2a1c 100644 --- a/src/server/errors.rs +++ b/src/server/errors.rs @@ -1,10 +1,11 @@ use crate::static_init::error::Error; -use std::fmt; +use std::{fmt, io}; use tokio::sync::broadcast::error::SendError; use warp::reject::Reject; #[derive(Debug)] pub enum Rejection { + IoRejection(String), SendRejection(String), } @@ -14,6 +15,7 @@ impl fmt::Display for Rejection { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> std::fmt::Result { match self { Rejection::SendRejection(e) => write!(f, "Send error: {}", e), + Rejection::IoRejection(e) => write!(f, "IO error: {}", e), } } } @@ -27,6 +29,12 @@ where } } +impl From for Rejection { + fn from(error: io::Error) -> Self { + Self::IoRejection(format!("{:?}", error)) + } +} + pub trait Rejectable { fn into_rejection(self) -> warp::Rejection; } @@ -46,19 +54,35 @@ impl Rejectable for Error { } } +impl Rejectable for io::Error { + fn into_rejection(self) -> warp::Rejection { + warp::reject::custom(Rejection::from(self)) + } +} + #[cfg(test)] mod tests { use super::*; + use std::io::ErrorKind; #[test] fn rejection_is_reject() { - let send_error = SendError(123); - let expected_message = "SendError(123)"; - match Rejection::from(send_error) { - Rejection::SendRejection(message) => { - assert_eq!(message, expected_message.to_string()); + fn check_rejection(rejection: Rejection, expected_message: &str) { + match rejection { + Rejection::SendRejection(message) => { + assert_eq!(message, expected_message.to_string()); + } + Rejection::IoRejection(message) => { + assert_eq!(message, expected_message.to_string()); + } } } + + check_rejection( + Rejection::from(io::Error::new(ErrorKind::AddrInUse, "test")), + "Custom { kind: AddrInUse, error: \"test\" }", + ); + check_rejection(Rejection::from(SendError(123)), "SendError(123)"); } mod display { diff --git a/src/static_init/error.rs b/src/static_init/error.rs index 32fe305..436211a 100644 --- a/src/static_init/error.rs +++ b/src/static_init/error.rs @@ -10,11 +10,9 @@ use std::fmt::{Display, Formatter}; use std::io; use tokio::task::JoinError; use warp::reject::Reject; -use Error::Builder; #[derive(Clone, Debug, PartialEq)] pub enum Error { - Builder(String), Cancelled(String), Connection(String), FailedAfterRetries, @@ -35,7 +33,6 @@ impl Reject for Error {} impl Display for Error { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { - Builder(e) => write!(f, "Builder error: {}", e), Error::Cancelled(e) => write!(f, "Cancelled: {}", e), Error::Connection(e) => write!(f, "Connection error: {}", e), Error::FailedAfterRetries => write!(f, "Failed after retries"), @@ -119,16 +116,21 @@ impl SerdeErrorExt for serde_yaml::Error { } pub trait IoErrorExt { - fn to_source_creation_builder_error(&self) -> Error; fn to_error(&self) -> Error; } impl IoErrorExt for io::Error { - fn to_source_creation_builder_error(&self) -> Error { - Builder(self.to_string()) - } - fn to_error(&self) -> Error { Io(self.to_string()) } } + +#[cfg(test)] +mod test { + #[test] + fn test_error_is_send_and_sync() { + fn is_send_and_sync(_candidate: &T) {} + + is_send_and_sync(&super::Error::Io("test".to_string())); + } +} diff --git a/tests/cucumber/google/client.rs b/tests/cucumber/google/client.rs new file mode 100644 index 0000000..dd64b6a --- /dev/null +++ b/tests/cucumber/google/client.rs @@ -0,0 +1,63 @@ +use cloud_scraper::domain::node::Manager; +use cloud_scraper::domain::oauth2::Client; +use cloud_scraper::domain::oauth2::{ApplicationSecret, ExtraParameters}; +use cloud_scraper::server::WebEventChannelHandle; +use cloud_scraper::static_init::error::Error; +use oauth2::AccessToken; +use parking_lot::ReentrantMutex; +use std::fmt::{Debug, Formatter}; +use std::path::Path; +use std::sync::Arc; + +pub(crate) struct TestClientImpl {} + +impl Debug for TestClientImpl { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "TestClientImpl") + } +} + +impl TestClientImpl { + fn new_proxy( + application_secret: ApplicationSecret, + extra_parameters: &ExtraParameters, + manager: &Manager, + token_path: &Path, + web_channel_handle: &WebEventChannelHandle, + ) -> Self { + Self {} + } + fn get_token_proxy(&self, scopes: &[String]) -> Result { + Ok(AccessToken::new("token".to_string())) + } +} + +#[derive(Clone, Debug)] +pub(crate) struct TestClient { + internal: Arc>, +} + +impl Client for TestClient { + fn new( + application_secret: ApplicationSecret, + extra_parameters: &ExtraParameters, + manager: &Manager, + token_path: &Path, + web_channel_handle: &WebEventChannelHandle, + ) -> Self { + Self { + internal: Arc::new(ReentrantMutex::new(TestClientImpl::new_proxy( + application_secret, + extra_parameters, + manager, + token_path, + web_channel_handle, + ))), + } + } + + async fn get_token(&self, scopes: &[&str]) -> Result { + let scopes = scopes.iter().map(|s| s.to_string()).collect::>(); + self.internal.lock().get_token_proxy(&scopes) + } +} diff --git a/tests/cucumber/google/mod.rs b/tests/cucumber/google/mod.rs new file mode 100644 index 0000000..1d33131 --- /dev/null +++ b/tests/cucumber/google/mod.rs @@ -0,0 +1 @@ +pub(crate) mod client; diff --git a/tests/cucumber/google_world.rs b/tests/cucumber/google_world.rs index 4f1406a..7ea9843 100644 --- a/tests/cucumber/google_world.rs +++ b/tests/cucumber/google_world.rs @@ -1,3 +1,4 @@ +use crate::google::client::TestClient; use cloud_scraper::domain::node::{LifecycleChannelHandle, Manager}; use cloud_scraper::domain::{one_shot, Config}; use cloud_scraper::integration::google::Source; @@ -42,7 +43,7 @@ pub(crate) struct GoogleWorld { #[getter(skip)] manager: Option, #[getter(skip)] - source: Option>, + source: Option>>, source_once: Once, web_channel_handle: WebEventChannelHandle, } @@ -74,7 +75,7 @@ impl GoogleWorld { async fn test<'a>( permit: OwnedSemaphorePermit, semaphore: &'a Arc, - source: &'a Arc, + source: &'a Arc>, ) { drop(permit); source @@ -170,7 +171,7 @@ impl GoogleWorld { self.manager.as_ref().unwrap() } - fn source(&mut self) -> Arc { + fn source(&mut self) -> Arc> { self.source_once.call_once(|| { let source = Source::new(&self.manager.as_ref().unwrap(), &self.web_channel_handle); diff --git a/tests/cucumber/main.rs b/tests/cucumber/main.rs index 526d8b3..647c3d2 100644 --- a/tests/cucumber/main.rs +++ b/tests/cucumber/main.rs @@ -1,6 +1,7 @@ extern crate core; mod cli_world; +mod google; mod google_bindings; mod google_world; mod shared; diff --git a/tests/features/cli/serve.feature b/tests/features/cli/serve.feature index 2188b11..c1f7b07 100644 --- a/tests/features/cli/serve.feature +++ b/tests/features/cli/serve.feature @@ -25,9 +25,8 @@ Feature: Serve subcommand \s*\[[\d]{4}-[\d]{2}-[\d]{2}T[\d]{2}:[\d]{2}:[\d]{2}Z DEBUG cloud_scraper::main_impl\] Starting engine """ And the stderr should have matched: - """\s*\[[\d]{4}-[\d]{2}-[\d]{2}T[\d]{2}:[\d]{2}:[\d]{2}Z DEBUG cloud_scraper::integration::google::auth::web\] Root: "state/google".* -\s*\[[\d]{4}-[\d]{2}-[\d]{2}T[\d]{2}:[\d]{2}:[\d]{2}Z DEBUG cloud_scraper::integration::google::auth::web\] Config path: "state/google/config\.yaml".* -\s*\[[\d]{4}-[\d]{2}-[\d]{2}T[\d]{2}:[\d]{2}:[\d]{2}Z DEBUG cloud_scraper::integration::google::auth::web\] Read result: Err\(Os \{ code: 2, kind: NotFound, message: "No such file or directory" \}\) + """\s*\[[\d]{4}-[\d]{2}-[\d]{2}T[\d]{2}:[\d]{2}:[\d]{2}Z DEBUG cloud_scraper::integration::google::source\] Root: "state/google".* +\s*\[[\d]{4}-[\d]{2}-[\d]{2}T[\d]{2}:[\d]{2}:[\d]{2}Z DEBUG cloud_scraper::integration::google::auth::config\] Config path: "state/google/config\.yaml".* """ And the stderr should have matched: """\s*\[[\d]{4}-[\d]{2}-[\d]{2}T[\d]{2}:[\d]{2}:[\d]{2}Z INFO cloud_scraper::integration::google::source\] Loading google source From 8ab2057b18f7546755273d800006e7db46cb4b4c Mon Sep 17 00:00:00 2001 From: Rob Williamson Date: Sat, 11 Jan 2025 23:39:19 +0100 Subject: [PATCH 09/13] Separate Functional Tests So they can depend on `mockall` without making the Cloud Scraper binary need to depend on it. --- .gitignore | 2 +- Cargo.toml | 57 +------------------ cloud scraper.iml | 6 +- cloud_scraper/Cargo.toml | 48 ++++++++++++++++ .../resources}/html/config/google.html | 0 .../resources}/html/index.html | 0 .../resources}/html/login.html | 0 .../resources}/js/redirect.js | 0 {src => cloud_scraper/src}/core/cli.rs | 0 .../src}/core/construct_config.rs | 0 {src => cloud_scraper/src}/core/engine.rs | 0 {src => cloud_scraper/src}/core/error.rs | 0 {src => cloud_scraper/src}/core/hash.rs | 0 {src => cloud_scraper/src}/core/mod.rs | 0 {src => cloud_scraper/src}/core/module.rs | 0 .../src}/core/node_handles.rs | 0 {src => cloud_scraper/src}/core/password.rs | 0 .../src}/core/root_password.rs | 0 {src => cloud_scraper/src}/core/serde_yaml.rs | 0 .../src}/domain/channel_handle.rs | 0 {src => cloud_scraper/src}/domain/config.rs | 0 {src => cloud_scraper/src}/domain/entity.rs | 0 .../src}/domain/entity_data/mod.rs | 0 .../domain/entity_data/string_entity_data.rs | 0 .../domain/entity_data/uuid_entity_data.rs | 0 {src => cloud_scraper/src}/domain/mod.rs | 0 .../src}/domain/module_state.rs | 0 .../src}/domain/mpsc_handle.rs | 0 .../src}/domain/node/manager.rs | 0 {src => cloud_scraper/src}/domain/node/mod.rs | 0 .../src}/domain/oauth2/application_secret.rs | 0 .../src}/domain/oauth2/client.rs | 0 .../src}/domain/oauth2/config.rs | 0 .../src}/domain/oauth2/extra_parameters.rs | 0 .../src}/domain/oauth2/mod.rs | 0 .../src}/domain/oauth2/token.rs | 0 .../src}/integration/google/auth/config.rs | 0 .../src}/integration/google/auth/delegate.rs | 0 .../src}/integration/google/auth/mod.rs | 0 .../src}/integration/google/auth/web.rs | 0 .../src}/integration/google/mod.rs | 0 .../src}/integration/google/source.rs | 0 .../src}/integration/google/tasks/mod.rs | 0 .../src}/integration/google/tasks/sync.rs | 0 .../src}/integration/log/mod.rs | 0 {src => cloud_scraper/src}/integration/mod.rs | 0 .../src}/integration/stub/mod.rs | 0 {src => cloud_scraper/src}/lib.rs | 0 {src => cloud_scraper/src}/macros/mod.rs | 0 {src => cloud_scraper/src}/macros/test.rs | 0 {src => cloud_scraper/src}/main.rs | 0 {src => cloud_scraper/src}/main_impl.rs | 0 .../server/acme/challenge_token_server.rs | 0 {src => cloud_scraper/src}/server/acme/mod.rs | 0 .../src}/server/acme/types.rs | 0 {src => cloud_scraper/src}/server/auth.rs | 0 {src => cloud_scraper/src}/server/errors.rs | 0 {src => cloud_scraper/src}/server/events.rs | 0 .../src}/server/javascript/mod.rs | 0 .../src}/server/javascript/redirect.rs | 0 {src => cloud_scraper/src}/server/mod.rs | 0 .../src}/server/oauth2/mod.rs | 0 .../src}/server/page/login.rs | 0 {src => cloud_scraper/src}/server/page/mod.rs | 0 {src => cloud_scraper/src}/server/root.rs | 0 {src => cloud_scraper/src}/server/routes.rs | 0 .../src}/server/site_state.rs | 0 .../src}/server/web_server.rs | 0 .../src}/server/websocket/mod.rs | 0 .../src}/server/websocket/result.rs | 0 .../src}/server/websocket/socket.rs | 0 .../src}/server/websocket/use_websocket.rs | 0 .../server/websocket/websocket_handler.rs | 0 .../src}/static_init/error.rs | 0 {src => cloud_scraper/src}/static_init/mod.rs | 0 {src => cloud_scraper/src}/test.rs | 0 .../tests}/fixtures/cert.pem | 0 .../tests}/fixtures/key.pem | 0 functional_tests/Cargo.toml | 21 +++++++ functional_tests/src/main.rs | 1 + .../tests}/cucumber/cli_world.rs | 39 +++++++------ .../tests}/cucumber/google/client.rs | 12 ++-- .../tests}/cucumber/google/mod.rs | 0 .../tests}/cucumber/google_bindings.rs | 0 .../tests}/cucumber/google_world.rs | 0 .../tests}/cucumber/main.rs | 0 functional_tests/tests/cucumber/shared.rs | 31 ++++++++++ .../tests}/features/cli/config.feature | 0 .../tests}/features/cli/serve.feature | 0 .../tests}/features/google/source.feature | 0 .../tests}/fixtures/empty_config.yaml | 0 tests/cucumber/shared.rs | 10 ---- tests/fixtures/config.yaml | 2 - 93 files changed, 136 insertions(+), 93 deletions(-) create mode 100644 cloud_scraper/Cargo.toml rename {resources => cloud_scraper/resources}/html/config/google.html (100%) rename {resources => cloud_scraper/resources}/html/index.html (100%) rename {resources => cloud_scraper/resources}/html/login.html (100%) rename {resources => cloud_scraper/resources}/js/redirect.js (100%) rename {src => cloud_scraper/src}/core/cli.rs (100%) rename {src => cloud_scraper/src}/core/construct_config.rs (100%) rename {src => cloud_scraper/src}/core/engine.rs (100%) rename {src => cloud_scraper/src}/core/error.rs (100%) rename {src => cloud_scraper/src}/core/hash.rs (100%) rename {src => cloud_scraper/src}/core/mod.rs (100%) rename {src => cloud_scraper/src}/core/module.rs (100%) rename {src => cloud_scraper/src}/core/node_handles.rs (100%) rename {src => cloud_scraper/src}/core/password.rs (100%) rename {src => cloud_scraper/src}/core/root_password.rs (100%) rename {src => cloud_scraper/src}/core/serde_yaml.rs (100%) rename {src => cloud_scraper/src}/domain/channel_handle.rs (100%) rename {src => cloud_scraper/src}/domain/config.rs (100%) rename {src => cloud_scraper/src}/domain/entity.rs (100%) rename {src => cloud_scraper/src}/domain/entity_data/mod.rs (100%) rename {src => cloud_scraper/src}/domain/entity_data/string_entity_data.rs (100%) rename {src => cloud_scraper/src}/domain/entity_data/uuid_entity_data.rs (100%) rename {src => cloud_scraper/src}/domain/mod.rs (100%) rename {src => cloud_scraper/src}/domain/module_state.rs (100%) rename {src => cloud_scraper/src}/domain/mpsc_handle.rs (100%) rename {src => cloud_scraper/src}/domain/node/manager.rs (100%) rename {src => cloud_scraper/src}/domain/node/mod.rs (100%) rename {src => cloud_scraper/src}/domain/oauth2/application_secret.rs (100%) rename {src => cloud_scraper/src}/domain/oauth2/client.rs (100%) rename {src => cloud_scraper/src}/domain/oauth2/config.rs (100%) rename {src => cloud_scraper/src}/domain/oauth2/extra_parameters.rs (100%) rename {src => cloud_scraper/src}/domain/oauth2/mod.rs (100%) rename {src => cloud_scraper/src}/domain/oauth2/token.rs (100%) rename {src => cloud_scraper/src}/integration/google/auth/config.rs (100%) rename {src => cloud_scraper/src}/integration/google/auth/delegate.rs (100%) rename {src => cloud_scraper/src}/integration/google/auth/mod.rs (100%) rename {src => cloud_scraper/src}/integration/google/auth/web.rs (100%) rename {src => cloud_scraper/src}/integration/google/mod.rs (100%) rename {src => cloud_scraper/src}/integration/google/source.rs (100%) rename {src => cloud_scraper/src}/integration/google/tasks/mod.rs (100%) rename {src => cloud_scraper/src}/integration/google/tasks/sync.rs (100%) rename {src => cloud_scraper/src}/integration/log/mod.rs (100%) rename {src => cloud_scraper/src}/integration/mod.rs (100%) rename {src => cloud_scraper/src}/integration/stub/mod.rs (100%) rename {src => cloud_scraper/src}/lib.rs (100%) rename {src => cloud_scraper/src}/macros/mod.rs (100%) rename {src => cloud_scraper/src}/macros/test.rs (100%) rename {src => cloud_scraper/src}/main.rs (100%) rename {src => cloud_scraper/src}/main_impl.rs (100%) rename {src => cloud_scraper/src}/server/acme/challenge_token_server.rs (100%) rename {src => cloud_scraper/src}/server/acme/mod.rs (100%) rename {src => cloud_scraper/src}/server/acme/types.rs (100%) rename {src => cloud_scraper/src}/server/auth.rs (100%) rename {src => cloud_scraper/src}/server/errors.rs (100%) rename {src => cloud_scraper/src}/server/events.rs (100%) rename {src => cloud_scraper/src}/server/javascript/mod.rs (100%) rename {src => cloud_scraper/src}/server/javascript/redirect.rs (100%) rename {src => cloud_scraper/src}/server/mod.rs (100%) rename {src => cloud_scraper/src}/server/oauth2/mod.rs (100%) rename {src => cloud_scraper/src}/server/page/login.rs (100%) rename {src => cloud_scraper/src}/server/page/mod.rs (100%) rename {src => cloud_scraper/src}/server/root.rs (100%) rename {src => cloud_scraper/src}/server/routes.rs (100%) rename {src => cloud_scraper/src}/server/site_state.rs (100%) rename {src => cloud_scraper/src}/server/web_server.rs (100%) rename {src => cloud_scraper/src}/server/websocket/mod.rs (100%) rename {src => cloud_scraper/src}/server/websocket/result.rs (100%) rename {src => cloud_scraper/src}/server/websocket/socket.rs (100%) rename {src => cloud_scraper/src}/server/websocket/use_websocket.rs (100%) rename {src => cloud_scraper/src}/server/websocket/websocket_handler.rs (100%) rename {src => cloud_scraper/src}/static_init/error.rs (100%) rename {src => cloud_scraper/src}/static_init/mod.rs (100%) rename {src => cloud_scraper/src}/test.rs (100%) rename {tests => cloud_scraper/tests}/fixtures/cert.pem (100%) rename {tests => cloud_scraper/tests}/fixtures/key.pem (100%) create mode 100644 functional_tests/Cargo.toml create mode 100644 functional_tests/src/main.rs rename {tests => functional_tests/tests}/cucumber/cli_world.rs (94%) rename {tests => functional_tests/tests}/cucumber/google/client.rs (84%) rename {tests => functional_tests/tests}/cucumber/google/mod.rs (100%) rename {tests => functional_tests/tests}/cucumber/google_bindings.rs (100%) rename {tests => functional_tests/tests}/cucumber/google_world.rs (100%) rename {tests => functional_tests/tests}/cucumber/main.rs (100%) create mode 100644 functional_tests/tests/cucumber/shared.rs rename {tests => functional_tests/tests}/features/cli/config.feature (100%) rename {tests => functional_tests/tests}/features/cli/serve.feature (100%) rename {tests => functional_tests/tests}/features/google/source.feature (100%) rename {tests => functional_tests/tests}/fixtures/empty_config.yaml (100%) delete mode 100644 tests/cucumber/shared.rs delete mode 100644 tests/fixtures/config.yaml diff --git a/.gitignore b/.gitignore index 4af549c..a30f9d1 100644 --- a/.gitignore +++ b/.gitignore @@ -99,4 +99,4 @@ lcov.info *.json config*.yaml state/ -*password* +*password*.yaml diff --git a/Cargo.toml b/Cargo.toml index 5be8738..0c94f17 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,55 +1,2 @@ -[package] -name = "cloud_scraper" -version = "0.1.0" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -acme2 = "0.5.1" -async-trait = "0.1.77" -chrono = { version = "0.4.35", features = ["serde"] } -clap = { version = "4.0", features = ["derive"] } -derive-getters = "0.5.0" -derive_builder = "0.20.1" -env_logger = "0.11.1" -google-tasks1 = "6.0.0" -handlebars = "6.0.0" -hex = "0.4.3" -hmac = "0.12.1" -hyper-util = "0.1.7" -lazy_static = "1.4.0" -log = "0.4.20" -oauth2 = "4.4.2" -parking_lot = "0.12.1" -paste = "1.0.15" -pbkdf2 = "0.12.2" -rand = "0.8.5" -rpassword = "7.3.1" -serde = { version = "1.0.193", features = ["derive"] } -serde_yaml = { version = "0.9.29", features = [] } -sha2 = "0.10.8" -tokio = { version = "1.35.1", features = ["macros", "process", "rt", "rt-multi-thread", "signal", "time"] } -tokio-stream = "0.1.15" -url = "2.5.2" -uuid = { version = "1.6.1", features = ["serde", "v4"] } -warp = { version = "0.3", features = ["tls"] } -x509-parser = "0.16.0" - -[dev-dependencies] -cucumber = "0.21.1" -mockall = "0.13.0" -once_cell = "1.19.0" -regex = "1.11.0" -reqwest = "0.12.9" -tokio-test = "0.4.3" - -[profile.release] -panic = "abort" - -[profile.dev] -panic = "abort" - -[[test]] -name = "cucumber" -harness = false +[workspace] +members = ["cloud_scraper", "functional_tests"] diff --git a/cloud scraper.iml b/cloud scraper.iml index d7b1edb..1f25272 100644 --- a/cloud scraper.iml +++ b/cloud scraper.iml @@ -3,8 +3,10 @@ - - + + + + diff --git a/cloud_scraper/Cargo.toml b/cloud_scraper/Cargo.toml new file mode 100644 index 0000000..6e1e71f --- /dev/null +++ b/cloud_scraper/Cargo.toml @@ -0,0 +1,48 @@ +[package] +name = "cloud_scraper" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +acme2 = "0.5.1" +async-trait = "0.1.77" +chrono = { version = "0.4.35", features = ["serde"] } +clap = { version = "4.0", features = ["derive"] } +derive-getters = "0.5.0" +derive_builder = "0.20.1" +env_logger = "0.11.1" +google-tasks1 = "6.0.0" +handlebars = "6.0.0" +hex = "0.4.3" +hmac = "0.12.1" +hyper-util = "0.1.7" +lazy_static = "1.4.0" +log = "0.4.20" +oauth2 = "4.4.2" +parking_lot = "0.12.1" +paste = "1.0.15" +pbkdf2 = "0.12.2" +rand = "0.8.5" +rpassword = "7.3.1" +serde = { version = "1.0.193", features = ["derive"] } +serde_yaml = { version = "0.9.29", features = [] } +sha2 = "0.10.8" +tokio = { version = "1.35.1", features = ["macros", "process", "rt", "rt-multi-thread", "signal", "time"] } +tokio-stream = "0.1.15" +url = "2.5.2" +uuid = { version = "1.6.1", features = ["serde", "v4"] } +warp = { version = "0.3", features = ["tls"] } +x509-parser = "0.16.0" + +[dev-dependencies] +mockall = "0.13.0" +once_cell = "1.19.0" +tokio-test = "0.4.3" + +[profile.release] +panic = "abort" + +[profile.dev] +panic = "abort" diff --git a/resources/html/config/google.html b/cloud_scraper/resources/html/config/google.html similarity index 100% rename from resources/html/config/google.html rename to cloud_scraper/resources/html/config/google.html diff --git a/resources/html/index.html b/cloud_scraper/resources/html/index.html similarity index 100% rename from resources/html/index.html rename to cloud_scraper/resources/html/index.html diff --git a/resources/html/login.html b/cloud_scraper/resources/html/login.html similarity index 100% rename from resources/html/login.html rename to cloud_scraper/resources/html/login.html diff --git a/resources/js/redirect.js b/cloud_scraper/resources/js/redirect.js similarity index 100% rename from resources/js/redirect.js rename to cloud_scraper/resources/js/redirect.js diff --git a/src/core/cli.rs b/cloud_scraper/src/core/cli.rs similarity index 100% rename from src/core/cli.rs rename to cloud_scraper/src/core/cli.rs diff --git a/src/core/construct_config.rs b/cloud_scraper/src/core/construct_config.rs similarity index 100% rename from src/core/construct_config.rs rename to cloud_scraper/src/core/construct_config.rs diff --git a/src/core/engine.rs b/cloud_scraper/src/core/engine.rs similarity index 100% rename from src/core/engine.rs rename to cloud_scraper/src/core/engine.rs diff --git a/src/core/error.rs b/cloud_scraper/src/core/error.rs similarity index 100% rename from src/core/error.rs rename to cloud_scraper/src/core/error.rs diff --git a/src/core/hash.rs b/cloud_scraper/src/core/hash.rs similarity index 100% rename from src/core/hash.rs rename to cloud_scraper/src/core/hash.rs diff --git a/src/core/mod.rs b/cloud_scraper/src/core/mod.rs similarity index 100% rename from src/core/mod.rs rename to cloud_scraper/src/core/mod.rs diff --git a/src/core/module.rs b/cloud_scraper/src/core/module.rs similarity index 100% rename from src/core/module.rs rename to cloud_scraper/src/core/module.rs diff --git a/src/core/node_handles.rs b/cloud_scraper/src/core/node_handles.rs similarity index 100% rename from src/core/node_handles.rs rename to cloud_scraper/src/core/node_handles.rs diff --git a/src/core/password.rs b/cloud_scraper/src/core/password.rs similarity index 100% rename from src/core/password.rs rename to cloud_scraper/src/core/password.rs diff --git a/src/core/root_password.rs b/cloud_scraper/src/core/root_password.rs similarity index 100% rename from src/core/root_password.rs rename to cloud_scraper/src/core/root_password.rs diff --git a/src/core/serde_yaml.rs b/cloud_scraper/src/core/serde_yaml.rs similarity index 100% rename from src/core/serde_yaml.rs rename to cloud_scraper/src/core/serde_yaml.rs diff --git a/src/domain/channel_handle.rs b/cloud_scraper/src/domain/channel_handle.rs similarity index 100% rename from src/domain/channel_handle.rs rename to cloud_scraper/src/domain/channel_handle.rs diff --git a/src/domain/config.rs b/cloud_scraper/src/domain/config.rs similarity index 100% rename from src/domain/config.rs rename to cloud_scraper/src/domain/config.rs diff --git a/src/domain/entity.rs b/cloud_scraper/src/domain/entity.rs similarity index 100% rename from src/domain/entity.rs rename to cloud_scraper/src/domain/entity.rs diff --git a/src/domain/entity_data/mod.rs b/cloud_scraper/src/domain/entity_data/mod.rs similarity index 100% rename from src/domain/entity_data/mod.rs rename to cloud_scraper/src/domain/entity_data/mod.rs diff --git a/src/domain/entity_data/string_entity_data.rs b/cloud_scraper/src/domain/entity_data/string_entity_data.rs similarity index 100% rename from src/domain/entity_data/string_entity_data.rs rename to cloud_scraper/src/domain/entity_data/string_entity_data.rs diff --git a/src/domain/entity_data/uuid_entity_data.rs b/cloud_scraper/src/domain/entity_data/uuid_entity_data.rs similarity index 100% rename from src/domain/entity_data/uuid_entity_data.rs rename to cloud_scraper/src/domain/entity_data/uuid_entity_data.rs diff --git a/src/domain/mod.rs b/cloud_scraper/src/domain/mod.rs similarity index 100% rename from src/domain/mod.rs rename to cloud_scraper/src/domain/mod.rs diff --git a/src/domain/module_state.rs b/cloud_scraper/src/domain/module_state.rs similarity index 100% rename from src/domain/module_state.rs rename to cloud_scraper/src/domain/module_state.rs diff --git a/src/domain/mpsc_handle.rs b/cloud_scraper/src/domain/mpsc_handle.rs similarity index 100% rename from src/domain/mpsc_handle.rs rename to cloud_scraper/src/domain/mpsc_handle.rs diff --git a/src/domain/node/manager.rs b/cloud_scraper/src/domain/node/manager.rs similarity index 100% rename from src/domain/node/manager.rs rename to cloud_scraper/src/domain/node/manager.rs diff --git a/src/domain/node/mod.rs b/cloud_scraper/src/domain/node/mod.rs similarity index 100% rename from src/domain/node/mod.rs rename to cloud_scraper/src/domain/node/mod.rs diff --git a/src/domain/oauth2/application_secret.rs b/cloud_scraper/src/domain/oauth2/application_secret.rs similarity index 100% rename from src/domain/oauth2/application_secret.rs rename to cloud_scraper/src/domain/oauth2/application_secret.rs diff --git a/src/domain/oauth2/client.rs b/cloud_scraper/src/domain/oauth2/client.rs similarity index 100% rename from src/domain/oauth2/client.rs rename to cloud_scraper/src/domain/oauth2/client.rs diff --git a/src/domain/oauth2/config.rs b/cloud_scraper/src/domain/oauth2/config.rs similarity index 100% rename from src/domain/oauth2/config.rs rename to cloud_scraper/src/domain/oauth2/config.rs diff --git a/src/domain/oauth2/extra_parameters.rs b/cloud_scraper/src/domain/oauth2/extra_parameters.rs similarity index 100% rename from src/domain/oauth2/extra_parameters.rs rename to cloud_scraper/src/domain/oauth2/extra_parameters.rs diff --git a/src/domain/oauth2/mod.rs b/cloud_scraper/src/domain/oauth2/mod.rs similarity index 100% rename from src/domain/oauth2/mod.rs rename to cloud_scraper/src/domain/oauth2/mod.rs diff --git a/src/domain/oauth2/token.rs b/cloud_scraper/src/domain/oauth2/token.rs similarity index 100% rename from src/domain/oauth2/token.rs rename to cloud_scraper/src/domain/oauth2/token.rs diff --git a/src/integration/google/auth/config.rs b/cloud_scraper/src/integration/google/auth/config.rs similarity index 100% rename from src/integration/google/auth/config.rs rename to cloud_scraper/src/integration/google/auth/config.rs diff --git a/src/integration/google/auth/delegate.rs b/cloud_scraper/src/integration/google/auth/delegate.rs similarity index 100% rename from src/integration/google/auth/delegate.rs rename to cloud_scraper/src/integration/google/auth/delegate.rs diff --git a/src/integration/google/auth/mod.rs b/cloud_scraper/src/integration/google/auth/mod.rs similarity index 100% rename from src/integration/google/auth/mod.rs rename to cloud_scraper/src/integration/google/auth/mod.rs diff --git a/src/integration/google/auth/web.rs b/cloud_scraper/src/integration/google/auth/web.rs similarity index 100% rename from src/integration/google/auth/web.rs rename to cloud_scraper/src/integration/google/auth/web.rs diff --git a/src/integration/google/mod.rs b/cloud_scraper/src/integration/google/mod.rs similarity index 100% rename from src/integration/google/mod.rs rename to cloud_scraper/src/integration/google/mod.rs diff --git a/src/integration/google/source.rs b/cloud_scraper/src/integration/google/source.rs similarity index 100% rename from src/integration/google/source.rs rename to cloud_scraper/src/integration/google/source.rs diff --git a/src/integration/google/tasks/mod.rs b/cloud_scraper/src/integration/google/tasks/mod.rs similarity index 100% rename from src/integration/google/tasks/mod.rs rename to cloud_scraper/src/integration/google/tasks/mod.rs diff --git a/src/integration/google/tasks/sync.rs b/cloud_scraper/src/integration/google/tasks/sync.rs similarity index 100% rename from src/integration/google/tasks/sync.rs rename to cloud_scraper/src/integration/google/tasks/sync.rs diff --git a/src/integration/log/mod.rs b/cloud_scraper/src/integration/log/mod.rs similarity index 100% rename from src/integration/log/mod.rs rename to cloud_scraper/src/integration/log/mod.rs diff --git a/src/integration/mod.rs b/cloud_scraper/src/integration/mod.rs similarity index 100% rename from src/integration/mod.rs rename to cloud_scraper/src/integration/mod.rs diff --git a/src/integration/stub/mod.rs b/cloud_scraper/src/integration/stub/mod.rs similarity index 100% rename from src/integration/stub/mod.rs rename to cloud_scraper/src/integration/stub/mod.rs diff --git a/src/lib.rs b/cloud_scraper/src/lib.rs similarity index 100% rename from src/lib.rs rename to cloud_scraper/src/lib.rs diff --git a/src/macros/mod.rs b/cloud_scraper/src/macros/mod.rs similarity index 100% rename from src/macros/mod.rs rename to cloud_scraper/src/macros/mod.rs diff --git a/src/macros/test.rs b/cloud_scraper/src/macros/test.rs similarity index 100% rename from src/macros/test.rs rename to cloud_scraper/src/macros/test.rs diff --git a/src/main.rs b/cloud_scraper/src/main.rs similarity index 100% rename from src/main.rs rename to cloud_scraper/src/main.rs diff --git a/src/main_impl.rs b/cloud_scraper/src/main_impl.rs similarity index 100% rename from src/main_impl.rs rename to cloud_scraper/src/main_impl.rs diff --git a/src/server/acme/challenge_token_server.rs b/cloud_scraper/src/server/acme/challenge_token_server.rs similarity index 100% rename from src/server/acme/challenge_token_server.rs rename to cloud_scraper/src/server/acme/challenge_token_server.rs diff --git a/src/server/acme/mod.rs b/cloud_scraper/src/server/acme/mod.rs similarity index 100% rename from src/server/acme/mod.rs rename to cloud_scraper/src/server/acme/mod.rs diff --git a/src/server/acme/types.rs b/cloud_scraper/src/server/acme/types.rs similarity index 100% rename from src/server/acme/types.rs rename to cloud_scraper/src/server/acme/types.rs diff --git a/src/server/auth.rs b/cloud_scraper/src/server/auth.rs similarity index 100% rename from src/server/auth.rs rename to cloud_scraper/src/server/auth.rs diff --git a/src/server/errors.rs b/cloud_scraper/src/server/errors.rs similarity index 100% rename from src/server/errors.rs rename to cloud_scraper/src/server/errors.rs diff --git a/src/server/events.rs b/cloud_scraper/src/server/events.rs similarity index 100% rename from src/server/events.rs rename to cloud_scraper/src/server/events.rs diff --git a/src/server/javascript/mod.rs b/cloud_scraper/src/server/javascript/mod.rs similarity index 100% rename from src/server/javascript/mod.rs rename to cloud_scraper/src/server/javascript/mod.rs diff --git a/src/server/javascript/redirect.rs b/cloud_scraper/src/server/javascript/redirect.rs similarity index 100% rename from src/server/javascript/redirect.rs rename to cloud_scraper/src/server/javascript/redirect.rs diff --git a/src/server/mod.rs b/cloud_scraper/src/server/mod.rs similarity index 100% rename from src/server/mod.rs rename to cloud_scraper/src/server/mod.rs diff --git a/src/server/oauth2/mod.rs b/cloud_scraper/src/server/oauth2/mod.rs similarity index 100% rename from src/server/oauth2/mod.rs rename to cloud_scraper/src/server/oauth2/mod.rs diff --git a/src/server/page/login.rs b/cloud_scraper/src/server/page/login.rs similarity index 100% rename from src/server/page/login.rs rename to cloud_scraper/src/server/page/login.rs diff --git a/src/server/page/mod.rs b/cloud_scraper/src/server/page/mod.rs similarity index 100% rename from src/server/page/mod.rs rename to cloud_scraper/src/server/page/mod.rs diff --git a/src/server/root.rs b/cloud_scraper/src/server/root.rs similarity index 100% rename from src/server/root.rs rename to cloud_scraper/src/server/root.rs diff --git a/src/server/routes.rs b/cloud_scraper/src/server/routes.rs similarity index 100% rename from src/server/routes.rs rename to cloud_scraper/src/server/routes.rs diff --git a/src/server/site_state.rs b/cloud_scraper/src/server/site_state.rs similarity index 100% rename from src/server/site_state.rs rename to cloud_scraper/src/server/site_state.rs diff --git a/src/server/web_server.rs b/cloud_scraper/src/server/web_server.rs similarity index 100% rename from src/server/web_server.rs rename to cloud_scraper/src/server/web_server.rs diff --git a/src/server/websocket/mod.rs b/cloud_scraper/src/server/websocket/mod.rs similarity index 100% rename from src/server/websocket/mod.rs rename to cloud_scraper/src/server/websocket/mod.rs diff --git a/src/server/websocket/result.rs b/cloud_scraper/src/server/websocket/result.rs similarity index 100% rename from src/server/websocket/result.rs rename to cloud_scraper/src/server/websocket/result.rs diff --git a/src/server/websocket/socket.rs b/cloud_scraper/src/server/websocket/socket.rs similarity index 100% rename from src/server/websocket/socket.rs rename to cloud_scraper/src/server/websocket/socket.rs diff --git a/src/server/websocket/use_websocket.rs b/cloud_scraper/src/server/websocket/use_websocket.rs similarity index 100% rename from src/server/websocket/use_websocket.rs rename to cloud_scraper/src/server/websocket/use_websocket.rs diff --git a/src/server/websocket/websocket_handler.rs b/cloud_scraper/src/server/websocket/websocket_handler.rs similarity index 100% rename from src/server/websocket/websocket_handler.rs rename to cloud_scraper/src/server/websocket/websocket_handler.rs diff --git a/src/static_init/error.rs b/cloud_scraper/src/static_init/error.rs similarity index 100% rename from src/static_init/error.rs rename to cloud_scraper/src/static_init/error.rs diff --git a/src/static_init/mod.rs b/cloud_scraper/src/static_init/mod.rs similarity index 100% rename from src/static_init/mod.rs rename to cloud_scraper/src/static_init/mod.rs diff --git a/src/test.rs b/cloud_scraper/src/test.rs similarity index 100% rename from src/test.rs rename to cloud_scraper/src/test.rs diff --git a/tests/fixtures/cert.pem b/cloud_scraper/tests/fixtures/cert.pem similarity index 100% rename from tests/fixtures/cert.pem rename to cloud_scraper/tests/fixtures/cert.pem diff --git a/tests/fixtures/key.pem b/cloud_scraper/tests/fixtures/key.pem similarity index 100% rename from tests/fixtures/key.pem rename to cloud_scraper/tests/fixtures/key.pem diff --git a/functional_tests/Cargo.toml b/functional_tests/Cargo.toml new file mode 100644 index 0000000..296e953 --- /dev/null +++ b/functional_tests/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "functional_tests" +version = "0.1.0" +edition = "2021" + +[dependencies] +cloud_scraper = { version = "0.1.0", path = "../cloud_scraper" } +cucumber = "0.21.1" +derive-getters = "0.5.0" +mockall = "0.13.1" +oauth2 = "4.4.2" +parking_lot = "0.12.3" +regex = "1.11.1" +reqwest = "0.12.12" +serde_yaml = "0.9.34" +tokio = "1.43.0" +tokio-test = "0.4.4" + +[[test]] +name = "cucumber" +harness = false diff --git a/functional_tests/src/main.rs b/functional_tests/src/main.rs new file mode 100644 index 0000000..f328e4d --- /dev/null +++ b/functional_tests/src/main.rs @@ -0,0 +1 @@ +fn main() {} diff --git a/tests/cucumber/cli_world.rs b/functional_tests/tests/cucumber/cli_world.rs similarity index 94% rename from tests/cucumber/cli_world.rs rename to functional_tests/tests/cucumber/cli_world.rs index 2b13e50..5e69dee 100644 --- a/tests/cucumber/cli_world.rs +++ b/functional_tests/tests/cucumber/cli_world.rs @@ -1,4 +1,4 @@ -use crate::shared::test_config; +use crate::shared::{bin_folder, cs_home_dir, default_config_file, test_config}; use cloud_scraper::domain::Config; use cucumber::gherkin::Step; use cucumber::{given, then, when, World}; @@ -9,7 +9,6 @@ use std::collections::HashMap; use std::fmt::Debug; use std::future::Future; use std::io; -use std::path::PathBuf; use std::pin::Pin; use std::process::Output; use std::time::Duration; @@ -197,7 +196,7 @@ impl CliWorld { } let command = self.command.clone().expect("Command not set"); - let command = PathBuf::new().join("target/debug").join(command); + let command = bin_folder().join(command); let cmd = command.clone(); let mut command = Command::new(cmd); @@ -304,40 +303,42 @@ impl CliWorld { #[given(regex = r#"no file named "([\S ]+)""#)] async fn no_file(_cli_world: &mut CliWorld, path: String) { + let path = cs_home_dir().join(path); if fs::try_exists(&path) .await .expect("Error checking file existence") { fs::remove_file(&path) .await - .expect(&format!("Error removing {}", path)); + .expect(&format!("Error removing {:?}", path)); } } #[given(regex = r#"a file named "([\S ]+)" containing:"#)] async fn a_file_containing(_cli_world: &mut CliWorld, step: &Step, path: String) { + let path = cs_home_dir().join(path); if fs::try_exists(&path) .await .expect("Error checking file existence") { fs::remove_file(&path) .await - .expect(&format!("Error removing {}", path)); + .expect(&format!("Error removing {:?}", path)); } fs::write(&path, step.docstring.as_ref().unwrap().as_bytes()) .await - .expect(&format!("Error writing to {}", path)); + .expect(&format!("Error writing to {:?}", path)); } #[given("a test config")] async fn a_config_file(_cli_world: &mut CliWorld) { - fs::write( - "config.yaml", - serde_yaml::to_string(&test_config()).unwrap(), - ) - .await - .expect("Error writing config file"); + let path = default_config_file(); + let failure_message = format!("Error writing config file to {:?}", path); + + fs::write(path, serde_yaml::to_string(&test_config()).unwrap()) + .await + .expect(&failure_message); } #[given(regex = r#"an environment variable "([\S ]+)" with the value "([\S ]+)""#)] @@ -391,9 +392,10 @@ pub(crate) async fn i_request_post_with_body(cli_world: &mut CliWorld, url: Stri #[then(regex = r#"^the file "([\S "]+)" should not exist$"#)] pub(crate) async fn the_file_should_not_exist(cli_world: &mut CliWorld, path: String) { cli_world.trigger().await; + let path = cs_home_dir().join(path); assert!( - !std::fs::exists(&path).expect(&format!("Error checking {} existence", &path)), - "File {} exists", + !std::fs::exists(&path).expect(&format!("Error checking {:?} existence", &path)), + "File {:?} exists", &path ); } @@ -401,9 +403,10 @@ pub(crate) async fn the_file_should_not_exist(cli_world: &mut CliWorld, path: St #[then(regex = r#"^the file "([\S "]+)" should exist$"#)] pub(crate) async fn the_file_should_exist(cli_world: &mut CliWorld, path: String) { cli_world.trigger().await; + let path = cs_home_dir().join(path); assert!( - std::fs::exists(&path).expect(&format!("Error checking {} existence", &path)), - "File {} does not exist", + std::fs::exists(&path).expect(&format!("Error checking {:?} existence", &path)), + "File {:?} does not exist", &path ); } @@ -438,6 +441,7 @@ pub(crate) async fn the_exit_code_should_not_be(cli_world: &mut CliWorld, expect #[then(regex = r#"^the file "([\S "]+)" should be a valid config$"#)] pub(crate) async fn the_file_should_be_a_valid_config(cli_world: &mut CliWorld, path: String) { cli_world.trigger().await; + let path = cs_home_dir().join(path); let config = tokio::fs::read_to_string(&path) .await .expect("Error reading config file"); @@ -448,6 +452,7 @@ pub(crate) async fn the_file_should_be_a_valid_config(cli_world: &mut CliWorld, #[then(regex = r#"^the file "([\S "]+)" should contain:$"#)] pub(crate) async fn the_file_should_contain(cli_world: &mut CliWorld, step: &Step, path: String) { cli_world.trigger().await; + let path = cs_home_dir().join(path); let config = tokio::fs::read_to_string(&path) .await .expect("Error reading config file"); @@ -511,7 +516,7 @@ pub(crate) async fn the_stderr_should_have_matched(cli_world: &mut CliWorld, ste pub(crate) async fn the_test_config_should_be_unchanged(cli_world: &mut CliWorld) { cli_world.trigger().await; let config = test_config(); - let actual = tokio::fs::read_to_string("config.yaml") + let actual = tokio::fs::read_to_string(default_config_file()) .await .expect("Error reading config file"); let actual = serde_yaml::from_str::(&actual).expect("Error parsing config"); diff --git a/tests/cucumber/google/client.rs b/functional_tests/tests/cucumber/google/client.rs similarity index 84% rename from tests/cucumber/google/client.rs rename to functional_tests/tests/cucumber/google/client.rs index dd64b6a..fe366ef 100644 --- a/tests/cucumber/google/client.rs +++ b/functional_tests/tests/cucumber/google/client.rs @@ -19,15 +19,15 @@ impl Debug for TestClientImpl { impl TestClientImpl { fn new_proxy( - application_secret: ApplicationSecret, - extra_parameters: &ExtraParameters, - manager: &Manager, - token_path: &Path, - web_channel_handle: &WebEventChannelHandle, + _application_secret: ApplicationSecret, + _extra_parameters: &ExtraParameters, + _manager: &Manager, + _token_path: &Path, + _web_channel_handle: &WebEventChannelHandle, ) -> Self { Self {} } - fn get_token_proxy(&self, scopes: &[String]) -> Result { + fn get_token_proxy(&self, _scopes: &[String]) -> Result { Ok(AccessToken::new("token".to_string())) } } diff --git a/tests/cucumber/google/mod.rs b/functional_tests/tests/cucumber/google/mod.rs similarity index 100% rename from tests/cucumber/google/mod.rs rename to functional_tests/tests/cucumber/google/mod.rs diff --git a/tests/cucumber/google_bindings.rs b/functional_tests/tests/cucumber/google_bindings.rs similarity index 100% rename from tests/cucumber/google_bindings.rs rename to functional_tests/tests/cucumber/google_bindings.rs diff --git a/tests/cucumber/google_world.rs b/functional_tests/tests/cucumber/google_world.rs similarity index 100% rename from tests/cucumber/google_world.rs rename to functional_tests/tests/cucumber/google_world.rs diff --git a/tests/cucumber/main.rs b/functional_tests/tests/cucumber/main.rs similarity index 100% rename from tests/cucumber/main.rs rename to functional_tests/tests/cucumber/main.rs diff --git a/functional_tests/tests/cucumber/shared.rs b/functional_tests/tests/cucumber/shared.rs new file mode 100644 index 0000000..aefa76c --- /dev/null +++ b/functional_tests/tests/cucumber/shared.rs @@ -0,0 +1,31 @@ +use cloud_scraper::domain::{Config, DomainConfig}; +use std::path::PathBuf; + +pub(crate) fn test_config() -> Config { + Config::with_all_properties( + Some(DomainConfig::new("http://test.domain:8080")), + Some("user@test.domain".to_string()), + None, + None, + ) +} + +fn workspace() -> PathBuf { + test_crate_root().join("..") +} + +pub(crate) fn test_crate_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) +} + +pub(crate) fn cs_home_dir() -> PathBuf { + test_crate_root() +} + +pub(crate) fn default_config_file() -> PathBuf { + cs_home_dir().join("config.yaml") +} + +pub(crate) fn bin_folder() -> PathBuf { + workspace().join("target").join("debug") +} diff --git a/tests/features/cli/config.feature b/functional_tests/tests/features/cli/config.feature similarity index 100% rename from tests/features/cli/config.feature rename to functional_tests/tests/features/cli/config.feature diff --git a/tests/features/cli/serve.feature b/functional_tests/tests/features/cli/serve.feature similarity index 100% rename from tests/features/cli/serve.feature rename to functional_tests/tests/features/cli/serve.feature diff --git a/tests/features/google/source.feature b/functional_tests/tests/features/google/source.feature similarity index 100% rename from tests/features/google/source.feature rename to functional_tests/tests/features/google/source.feature diff --git a/tests/fixtures/empty_config.yaml b/functional_tests/tests/fixtures/empty_config.yaml similarity index 100% rename from tests/fixtures/empty_config.yaml rename to functional_tests/tests/fixtures/empty_config.yaml diff --git a/tests/cucumber/shared.rs b/tests/cucumber/shared.rs deleted file mode 100644 index 2523977..0000000 --- a/tests/cucumber/shared.rs +++ /dev/null @@ -1,10 +0,0 @@ -use cloud_scraper::domain::{Config, DomainConfig}; - -pub fn test_config() -> Config { - Config::with_all_properties( - Some(DomainConfig::new("http://test.domain:8080")), - Some("user@test.domain".to_string()), - None, - None, - ) -} diff --git a/tests/fixtures/config.yaml b/tests/fixtures/config.yaml deleted file mode 100644 index e5bc548..0000000 --- a/tests/fixtures/config.yaml +++ /dev/null @@ -1,2 +0,0 @@ -email: test@test.com -exit_after: 2 From 4b229ff67c623263e03bbaaa22866cfc9c5a125e Mon Sep 17 00:00:00 2001 From: Rob Williamson Date: Sun, 12 Jan 2025 11:08:54 +0100 Subject: [PATCH 10/13] Solve Execution Failure in CI The binary path for execution is wrong only in the CI build for coverage. Use resolver behaviour "2". Move the profiles from the cloud_scraper bin to the workspace. Expose the binary path when spawning a child process fails. --- Cargo.toml | 7 +++++++ cloud_scraper/Cargo.toml | 6 ------ functional_tests/tests/cucumber/cli_world.rs | 3 ++- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0c94f17..17812a6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,2 +1,9 @@ [workspace] members = ["cloud_scraper", "functional_tests"] +resolver = "2" + +[profile.release] +panic = "abort" + +[profile.dev] +panic = "abort" diff --git a/cloud_scraper/Cargo.toml b/cloud_scraper/Cargo.toml index 6e1e71f..d4feeb2 100644 --- a/cloud_scraper/Cargo.toml +++ b/cloud_scraper/Cargo.toml @@ -40,9 +40,3 @@ x509-parser = "0.16.0" mockall = "0.13.0" once_cell = "1.19.0" tokio-test = "0.4.3" - -[profile.release] -panic = "abort" - -[profile.dev] -panic = "abort" diff --git a/functional_tests/tests/cucumber/cli_world.rs b/functional_tests/tests/cucumber/cli_world.rs index 5e69dee..27ced62 100644 --- a/functional_tests/tests/cucumber/cli_world.rs +++ b/functional_tests/tests/cucumber/cli_world.rs @@ -197,6 +197,7 @@ impl CliWorld { let command = self.command.clone().expect("Command not set"); let command = bin_folder().join(command); + let spawning_error_message = format!("Error spawning command {:?}", command); let cmd = command.clone(); let mut command = Command::new(cmd); @@ -210,7 +211,7 @@ impl CliWorld { command = command.env(key, value); } - let mut child = command.spawn().expect("Error spawning command"); + let mut child = command.spawn().expect(&spawning_error_message); let mut stdin = child.stdin.take().expect("Error taking stdin"); From 553b20faaac3c2cffdc83a61f63f47740770ade8 Mon Sep 17 00:00:00 2001 From: Rob Williamson Date: Sun, 12 Jan 2025 11:22:03 +0100 Subject: [PATCH 11/13] Build the App Under Test Before Measuring Coverage This hadn't been happening. --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b6432d2..faa0d54 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,6 +33,11 @@ jobs: components: llvm-tools-preview override: true toolchain: stable + - name: build + uses: actions-rs/cargo@v1 + with: + command: build + args: --all-features - name: test with coverage env: CARGO_INCREMENTAL: '0' From 053bcab072556640135f6d43f7c710597507544e Mon Sep 17 00:00:00 2001 From: Rob Williamson Date: Sun, 12 Jan 2025 12:09:04 +0100 Subject: [PATCH 12/13] Remove Functional Test Main Added by the cargo template. --- functional_tests/src/main.rs | 1 - 1 file changed, 1 deletion(-) delete mode 100644 functional_tests/src/main.rs diff --git a/functional_tests/src/main.rs b/functional_tests/src/main.rs deleted file mode 100644 index f328e4d..0000000 --- a/functional_tests/src/main.rs +++ /dev/null @@ -1 +0,0 @@ -fn main() {} From 2174e3308e4a25df1550fc36a90f270cfdfb1930 Mon Sep 17 00:00:00 2001 From: Rob Williamson Date: Sun, 26 Jan 2025 22:10:00 +0100 Subject: [PATCH 13/13] WIP Make Oauth2 Client & Config Pluggable So test dependencies can be injected into the Google integration. Doing this statically with generics failed because of https://github.com/rust-lang/rust/issues/100013. Refactor to used boxed vtable-dispatching types. It's much less elegant. --- cloud_scraper/Cargo.toml | 1 + cloud_scraper/src/core/engine.rs | 16 +- cloud_scraper/src/domain/module_state.rs | 11 +- .../src/domain/oauth2/application_secret.rs | 10 + cloud_scraper/src/domain/oauth2/client.rs | 153 ++++++++-- cloud_scraper/src/domain/oauth2/config.rs | 288 +++++++++++------- cloud_scraper/src/domain/oauth2/mod.rs | 2 +- .../src/integration/google/auth/config.rs | 170 ++++------- .../src/integration/google/auth/delegate.rs | 31 +- .../src/integration/google/auth/mod.rs | 1 - .../src/integration/google/auth/web.rs | 74 ++++- .../src/integration/google/source.rs | 61 ++-- .../src/integration/google/tasks/sync.rs | 6 +- cloud_scraper/src/test.rs | 8 + .../tests/cucumber/google_world.rs | 4 +- 15 files changed, 533 insertions(+), 303 deletions(-) diff --git a/cloud_scraper/Cargo.toml b/cloud_scraper/Cargo.toml index d4feeb2..4f508a1 100644 --- a/cloud_scraper/Cargo.toml +++ b/cloud_scraper/Cargo.toml @@ -39,4 +39,5 @@ x509-parser = "0.16.0" [dev-dependencies] mockall = "0.13.0" once_cell = "1.19.0" +tempfile = "3.15.0" tokio-test = "0.4.3" diff --git a/cloud_scraper/src/core/engine.rs b/cloud_scraper/src/core/engine.rs index 6bc8938..1d67b99 100644 --- a/cloud_scraper/src/core/engine.rs +++ b/cloud_scraper/src/core/engine.rs @@ -9,7 +9,8 @@ use crate::core::node_handles::NodeHandles; use crate::domain::config::Config; use crate::domain::mpsc_handle::{one_shot, OneshotMpscSenderHandle}; use crate::domain::node::{LifecycleChannelHandle, Manager}; -use crate::domain::oauth2::BasicClientImpl; +use crate::domain::oauth2::{ApplicationSecret, ExtraParameters}; +use crate::domain::oauth2::{BasicClientImpl, Client}; use crate::integration::google::Source as GoogleSource; use crate::integration::log::Sink as LogSink; use crate::integration::stub::Source as StubSource; @@ -17,6 +18,7 @@ use crate::server::WebServer; use core::time::Duration; #[cfg(test)] use mockall::automock; +use std::path::Path; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering::SeqCst; use tokio::sync::Semaphore; @@ -67,7 +69,7 @@ where let wait_duration = self.manager.core_config().exit_after(); let mut stub_source = StubSource::new(&self.manager); - let google_source: GoogleSource = + let google_source: GoogleSource = GoogleSource::new(&self.manager, self.server.get_web_channel_handle()); let mut log_sink = LogSink::new(&self.manager, &stub_source.get_readonly_channel_handle()); @@ -98,7 +100,15 @@ where .expect("Could not acquire semaphore"); abort_handles.push(join_set.spawn(async move { log_sink.run(log_permit).await })); abort_handles.push(join_set.spawn(async move { stub_source.run(stub_permit).await })); - abort_handles.push(join_set.spawn(async move { google_source.run(google_permit).await })); + abort_handles.push(join_set.spawn(async move { + google_source + .run( + google_permit, + BasicClientImpl::get_auth_config, + BasicClientImpl::new, + ) + .await + })); let server = self.server.clone(); abort_handles.push(join_set.spawn(async move { diff --git a/cloud_scraper/src/domain/module_state.rs b/cloud_scraper/src/domain/module_state.rs index cada32e..1d9fa90 100644 --- a/cloud_scraper/src/domain/module_state.rs +++ b/cloud_scraper/src/domain/module_state.rs @@ -1,15 +1,20 @@ use async_trait::async_trait; +use std::path::PathBuf; use tokio::fs; #[async_trait] pub trait ModuleState { fn path() -> &'static str; - async fn path_for() -> Result + async fn path_for() -> Result where Module: NamedModule, { - let path = format!("{}/{}", Self::path(), Module::name()); - fs::create_dir_all(path.clone()).await?; + Self::path_for_name(Module::name()).await + } + + async fn path_for_name(name: &str) -> Result { + let path = PathBuf::from(Self::path()).join(name); + fs::create_dir_all(&path).await?; Ok(path) } } diff --git a/cloud_scraper/src/domain/oauth2/application_secret.rs b/cloud_scraper/src/domain/oauth2/application_secret.rs index 409794c..3c25616 100644 --- a/cloud_scraper/src/domain/oauth2/application_secret.rs +++ b/cloud_scraper/src/domain/oauth2/application_secret.rs @@ -39,3 +39,13 @@ impl ApplicationSecret { .set_redirect_uri(redirect_uri) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::assert_is_send_and_sync; + + fn send_and_sync() { + assert_is_send_and_sync!(ApplicationSecret); + } +} diff --git a/cloud_scraper/src/domain/oauth2/client.rs b/cloud_scraper/src/domain/oauth2/client.rs index 8f036cd..75c7d1e 100644 --- a/cloud_scraper/src/domain/oauth2/client.rs +++ b/cloud_scraper/src/domain/oauth2/client.rs @@ -1,8 +1,11 @@ +use crate::core::module::State; +use crate::domain::module_state::ModuleState; use crate::domain::mpsc_handle::one_shot; use crate::domain::node::Manager; use crate::domain::oauth2::extra_parameters::{ExtraParameters, WithExtraParametersExt}; use crate::domain::oauth2::token::{BasicTokenResponseExt, Token, TokenExt, TokenStatus}; -use crate::domain::oauth2::ApplicationSecret; +use crate::domain::oauth2::{ApplicationSecret, Config, PersistableConfig}; +use crate::integration::google::auth::ConfigQuery; use crate::server::Event::Redirect; use crate::server::{Code, Event, WebEventChannelHandle}; use crate::static_init::error::Error::FailedAfterRetries; @@ -18,7 +21,9 @@ use oauth2::{ RefreshToken, Scope, }; use std::future::Future; +use std::io; use std::path::{Path, PathBuf}; +use std::pin::Pin; use std::sync::Arc; use tokio::fs; use tokio::sync::broadcast::error::RecvError; @@ -28,18 +33,28 @@ use tokio::time::sleep; use Error::Oauth2CsrfMismatch; use Event::Oauth2Code; -pub trait Client: Clone + Send + Sized + Sync + 'static { +pub trait Client: Send + Sync + 'static { fn new( application_secret: ApplicationSecret, extra_parameters: &ExtraParameters, manager: &Manager, token_path: &Path, web_channel_handle: &WebEventChannelHandle, - ) -> Self; - fn get_token( - &self, - scopes: &[&str], - ) -> impl Future> + Send + Sync; + ) -> Pin> + where + Self: Sized + 'static; + fn get_auth_config<'async_trait>( + name: &'async_trait str, + ) -> Pin> + Send + 'async_trait>> + where + Self: Sized + Sync + 'async_trait; + fn duplicate(&self) -> Pin>; + fn get_token<'async_trait>( + &'async_trait self, + scopes: &'async_trait [&'async_trait str], + ) -> Pin> + Send + 'async_trait>> + where + Self: Sync + 'async_trait; } #[derive(Clone)] @@ -60,9 +75,9 @@ impl Client for BasicClientImpl { manager: &Manager, token_path: &Path, web_channel_handle: &WebEventChannelHandle, - ) -> Self { + ) -> Pin> { let basic_client = application_secret.to_client(); - Self { + Box::pin(Self { basic_client, extra_parameters: extra_parameters.clone(), manager: manager.clone(), @@ -70,21 +85,52 @@ impl Client for BasicClientImpl { retry_period: std::time::Duration::from_secs(2), token_path: token_path.to_owned(), web_channel_handle: web_channel_handle.clone(), - } + }) } - async fn get_token(&self, scopes: &[&str]) -> Result { - match self.get_token_status_from_file().await { - TokenStatus::Ok(token) => Ok(token.access_token().clone()), - TokenStatus::Expired(refresh_token) => self - .refresh_token(&refresh_token) - .await - .map(|token| token.access_token().clone()), - TokenStatus::Absent => self.retrieve_token(scopes).await.map(|token| { - debug!("Token retrieved: {:?}", token); - token.access_token().clone() - }), - } + fn get_auth_config<'async_trait>( + name: &'async_trait str, + ) -> Pin> + Send + 'async_trait>> + where + Self: Sized + Sync + 'async_trait, + { + Box::pin( + async move { Ok(ConfigQuery::read_config(&State::path_for_name(name).await?).await?) }, + ) + } + + fn duplicate(&self) -> Pin> { + Box::pin(Self { + basic_client: self.basic_client.clone(), + extra_parameters: self.extra_parameters.clone(), + manager: self.manager.clone(), + retry_max: self.retry_max, + retry_period: self.retry_period, + token_path: self.token_path.to_owned(), + web_channel_handle: self.web_channel_handle.clone(), + }) + } + + fn get_token<'async_trait>( + &'async_trait self, + scopes: &'async_trait [&'async_trait str], + ) -> Pin> + Send + 'async_trait>> + where + Self: Sync + 'async_trait, + { + Box::pin(async move { + match self.get_token_status_from_file().await { + TokenStatus::Ok(token) => Ok(token.access_token().clone()), + TokenStatus::Expired(refresh_token) => self + .refresh_token(&refresh_token) + .await + .map(|token| token.access_token().clone()), + TokenStatus::Absent => self.retrieve_token(scopes).await.map(|token| { + debug!("Token retrieved: {:?}", token); + token.access_token().clone() + }), + } + }) } } @@ -300,6 +346,16 @@ impl BasicClientImpl { pub mod tests { use super::*; + mod send_and_sync { + use super::*; + use crate::assert_is_send_and_sync; + + #[test] + fn basic_client_impl_is_send_and_sync() { + assert_is_send_and_sync!(BasicClientImpl); + } + } + mod make_redirect_url { use super::*; use crate::domain::config::tests::test_config; @@ -381,12 +437,61 @@ pub mod tests { } mod access_token { + use crate::assert_is_send_and_sync; use oauth2::AccessToken; #[test] fn test_is_send_and_sync() { - fn is_send_and_sync() {} - is_send_and_sync::(); + assert_is_send_and_sync!(AccessToken); + } + } + + mod get_auth_config { + use super::*; + use crate::domain::config::Config as CoreConfig; + use crate::domain::module_state::NamedModule; + use crate::domain::oauth2::config::ConfigProperties; + + pub struct NamedType; + + impl NamedModule for NamedType { + fn name() -> &'static str { + "test" + } + } + + impl NamedType { + async fn typed_test(&self) -> ApplicationSecret { + task::spawn(async move { + let config = T::get_auth_config("name").await.unwrap(); + assert_eq!(config.auth_uri(), "auth_uri"); + assert_eq!( + config.auth_provider_x509_cert_url(), + "auth_provider_x509_cert_url" + ); + assert_eq!(config.client_email(), Some("client_email")); + assert_eq!(config.client_id(), "client_id"); + assert_eq!(config.client_secret(), "client_secret"); + assert_eq!(config.client_x509_cert_url(), Some("client_x509_cert_url")); + assert_eq!(config.project_id(), "project_id"); + assert_eq!(config.redirect_uris(), &vec!["redirect_uris".to_string()]); + assert_eq!(config.token_uri(), "token_uri"); + let core_config = CoreConfig::with_all_properties(None, None, None, None); + let app_secret = config.to_application_secret(&core_config); + app_secret + }) + .await + .unwrap() + } + } + + // Trying to reproduce + // lifetime bound not satisfied + // Note: this is a known limitation that will be removed in the future (see issue #100013 for more information) + #[tokio::test] + async fn test_get_auth_config() { + let named_type = NamedType {}; + let _app_secret = named_type.typed_test::().await; } } } diff --git a/cloud_scraper/src/domain/oauth2/config.rs b/cloud_scraper/src/domain/oauth2/config.rs index af94b78..1ab5aaf 100644 --- a/cloud_scraper/src/domain/oauth2/config.rs +++ b/cloud_scraper/src/domain/oauth2/config.rs @@ -1,132 +1,202 @@ -use paste::paste; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; +use crate::domain; +use crate::domain::oauth2::{ApplicationSecret, ApplicationSecretBuilder}; +use async_trait::async_trait; +use log::debug; +use serde::de::DeserializeOwned; +use serde::Serialize; use std::fmt::Debug; -use std::future::Future; +use std::io; +use std::io::Error; use std::path::Path; +use tokio::fs; -macro_rules! make_config { - ($the_trait:ident, { $($e:ident),* }) => { - paste! { - pub trait $the_trait: Debug + Deserialize<'static> + From<&'static HashMap> + Serialize { - $( - fn $e(&self) -> &str; - )* - } - } - }; +pub trait ConfigProperties { + fn auth_provider_x509_cert_url(&self) -> &str; + fn auth_uri(&self) -> &str; + fn client_email(&self) -> Option<&str>; + fn client_id(&self) -> &str; + fn client_secret(&self) -> &str; + fn client_x509_cert_url(&self) -> Option<&str>; + fn project_id(&self) -> &str; + fn redirect_uris(&self) -> &Vec; + fn token_uri(&self) -> &str; +} + +pub trait Config: + ConfigProperties + Clone + Debug + DeserializeOwned + PartialEq + Send + Sync + Serialize +{ + fn to_application_secret(self, _core_config: &domain::Config) -> ApplicationSecret { + ApplicationSecretBuilder::default() + .auth_provider_x509_cert_url(Some(self.auth_provider_x509_cert_url().into())) + .auth_uri(self.auth_uri().to_string()) + .client_email(self.client_email().map(|s| s.to_string())) + .client_id(self.client_id().to_string()) + .client_secret(self.client_secret().to_string()) + .client_x509_cert_url(self.client_x509_cert_url().map(|s| s.to_string())) + .project_id(Some(self.project_id().to_string())) + .redirect_uris(self.redirect_uris().to_vec()) + .token_uri(self.token_uri().to_string()) + .build() + .unwrap_or_else(|e| { + panic!("Error while building ApplicationSecret: {:?}", e); + }) + } +} + +#[async_trait] +pub trait PersistableConfig: Sized { + async fn persist(&self, path: &Path) -> Result<(), Error>; + async fn read_config(path: &Path) -> Result; +} + +#[async_trait] +impl PersistableConfig for T +where + T: Config, +{ + async fn persist(&self, path: &Path) -> Result<(), Error> { + debug!("Config path: {:?}", path); + let serialized = serde_yaml::to_string(self).map_err(|e| { + Error::new( + io::ErrorKind::InvalidData, + format!("Could not serialize config file due to {:?}.", e), + ) + })?; + + fs::write(&path, serialized).await?; + + Ok(()) + } + + async fn read_config(path: &Path) -> Result { + debug!("Config path: {:?}", path); + let slice = fs::read(&path).await.map_err(|e| { + debug!( + "Could not read config file at {} due to {:?}.", + path.display(), + e + ); + e + })?; + debug!("Read result: {:?}", slice); + let config_query = serde_yaml::from_slice(&slice).map_err(|e| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("Could not parse config file due to {:?}.", e), + ) + })?; + debug!("Parse result: {:?}", config_query); + Ok(config_query) + } } -macro_rules! make_config_struct { - ($struct:ident, $the_trait:ident, { $($e:ident),* }, { $($d:ident, $v:literal),* }) => { - paste! { - #[derive(Builder, Debug, Deserialize, Serialize)] - pub struct $struct { - $( - $e: String, - )* - $( - $d: String, - )* - } - - impl $the_trait for $struct { - $( - fn $e(&self) -> &str { - &self.$e - } - )* - - $( - fn $d(&self) -> &str { - &self.$d - } - )* - } - - impl From<&HashMap> for $struct { - fn from(map: &HashMap) -> Self { - Self { - $( - $e: map.get(stringify!($e)).expect(&Self::format_missing_hash_key_message(stringify!($e), map)).clone(), - )* - $( - $d: map.get(stringify!($d)).unwrap_or(&String::from($v)).clone(), - )* - } - } - } - - impl $struct { - fn format_missing_hash_key_message(key: &str, map: &HashMap) -> String { - format!("Could not get required value {} from {:?}", key, map) - } - } +#[macro_export] +macro_rules! declare_config_struct { + ($struct:ident) => { + #[derive(Builder, Clone, Debug, Deserialize, PartialEq, Serialize)] + pub struct $struct { + auth_uri: String, + auth_provider_x509_cert_url: String, + client_email: Option, + client_id: String, + client_secret: String, + client_x509_cert_url: Option, + project_id: String, + redirect_uris: Vec, + token_uri: String, } + + impl Config for $struct {} }; } -pub(crate) use make_config_struct; - -make_config!( - Config, - { - auth_uri, - auth_provider_x509_cert_url, - client_id, - client_secret, - project_id, - token_uri - } -); - -pub trait PersistableConfig: Config { - fn persist( - &self, - path: &Path, - ) -> impl Future> + Send + Sync; - fn read_config(path: &Path) - -> impl Future> + Send + Sync; +#[macro_export] +macro_rules! implement_getter { + ($name:ident, $type:ty) => { + fn $name(&self) -> &$type { + &self.$name + } + }; } #[cfg(test)] -mod test { +pub mod tests { use super::*; use derive_builder::Builder; + use serde::Deserialize; - make_config!( - TestConfig, - { a, b } - ); - - make_config_struct!( - TestConfigStruct, TestConfig, { a }, { b, "c" } - ); + declare_config_struct!(TestConfig); + impl ConfigProperties for TestConfig { + implement_getter!(auth_uri, str); + implement_getter!(auth_provider_x509_cert_url, str); + fn client_email(&self) -> Option<&str> { + self.client_email.as_ref().map(|s| s.as_str()) + } + implement_getter!(client_id, str); + implement_getter!(client_secret, str); + fn client_x509_cert_url(&self) -> Option<&str> { + self.client_x509_cert_url.as_ref().map(|s| s.as_str()) + } + implement_getter!(project_id, str); + implement_getter!(redirect_uris, Vec); + implement_getter!(token_uri, str); + } #[test] - fn constructor_works() { - let config: TestConfigStruct = TestConfigStructBuilder::default() - .a("a".into()) - .b("b".into()) + fn test_config_properties() { + let config = TestConfigBuilder::default() + .auth_uri("auth_uri".to_string()) + .auth_provider_x509_cert_url("auth_provider_x509_cert_url".to_string()) + .client_email(Some("client_email".to_string())) + .client_id("client_id".to_string()) + .client_secret("client_secret".to_string()) + .client_x509_cert_url(Some("client_x509_cert_url".to_string())) + .project_id("project_id".to_string()) + .redirect_uris(vec!["redirect_uris".to_string()]) + .token_uri("token_uri".to_string()) .build() - .expect("Failed to build TestConfigStruct"); - assert_eq!(config.a(), "a"); - assert_eq!(config.b(), "b"); + .unwrap(); + assert_eq!(config.auth_uri(), "auth_uri"); + assert_eq!( + config.auth_provider_x509_cert_url(), + "auth_provider_x509_cert_url" + ); + assert_eq!(config.client_email(), Some("client_email")); + assert_eq!(config.client_id(), "client_id"); + assert_eq!(config.client_secret(), "client_secret"); + assert_eq!(config.client_x509_cert_url(), Some("client_x509_cert_url")); + assert_eq!(config.project_id(), "project_id"); + assert_eq!(config.redirect_uris(), &vec!["redirect_uris".to_string()]); + assert_eq!(config.token_uri(), "token_uri"); } - #[test] - fn from_hash_map_works() { - let mut map = HashMap::new(); - map.insert("a".into(), "a".into()); - let config: TestConfigStruct = (&map).into(); - assert_eq!(config.a(), "a"); - assert_eq!(config.b(), "c"); - } + mod persistable_config { + use crate::domain::oauth2::PersistableConfig; + use tempfile; - #[test] - #[should_panic] - fn from_hash_map_missing_key() { - let map = HashMap::new(); - let _config: TestConfigStruct = (&map).into(); + #[tokio::test] + async fn test_persistence() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.yaml"); + let config = super::TestConfigBuilder::default() + .auth_uri("auth_uri".to_string()) + .auth_provider_x509_cert_url("auth_provider_x509_cert_url".to_string()) + .client_email(Some("client_email".to_string())) + .client_id("client_id".to_string()) + .client_secret("client_secret".to_string()) + .client_x509_cert_url(Some("client_x509_cert_url".to_string())) + .project_id("project_id".to_string()) + .redirect_uris(vec!["redirect_uris".to_string()]) + .token_uri("token_uri".to_string()) + .build() + .unwrap(); + let result = config.persist(&path).await; + assert!(result.is_ok()); + + let read_config = super::TestConfig::read_config(&path) + .await + .expect("Could not read config"); + assert_eq!(config, read_config); + } } } diff --git a/cloud_scraper/src/domain/oauth2/mod.rs b/cloud_scraper/src/domain/oauth2/mod.rs index 4dd9983..c91d140 100644 --- a/cloud_scraper/src/domain/oauth2/mod.rs +++ b/cloud_scraper/src/domain/oauth2/mod.rs @@ -10,5 +10,5 @@ pub use application_secret::{ApplicationSecret, ApplicationSecretBuilder}; pub(crate) use client::BasicClientImpl; pub use client::Client; -pub(crate) use config::{make_config_struct, Config, PersistableConfig}; +pub(crate) use config::{Config, ConfigProperties, PersistableConfig}; pub use extra_parameters::{extra_parameters, ExtraParameter, ExtraParameters}; diff --git a/cloud_scraper/src/integration/google/auth/config.rs b/cloud_scraper/src/integration/google/auth/config.rs index 852b73f..d240804 100644 --- a/cloud_scraper/src/integration/google/auth/config.rs +++ b/cloud_scraper/src/integration/google/auth/config.rs @@ -1,122 +1,86 @@ -use crate::domain::oauth2::Config as Oauth2Config; -use crate::domain::oauth2::{ - make_config_struct, ApplicationSecret, ApplicationSecretBuilder, PersistableConfig, -}; -use crate::domain::Config; +use crate::domain::oauth2::{Config, ConfigProperties}; use derive_builder::Builder; -use log::debug; -use paste::paste; +use derive_getters::Getters; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; -use std::io; -use std::path::Path; -use tokio::fs; -macro_rules! make_config_query { - ($struct:ident, { $($e:ident),* }, { $($d:ident, $v:literal),* }) => { - make_config_struct!( - $struct, - Oauth2Config, - { $($e),* }, - { $($d, $v),* } - ); - paste! { - impl $struct { - pub(crate) fn empty_page_data() -> HashMap<&'static str, String> { - let mut page_data = HashMap::new(); - $( - page_data.insert(stringify!($e), Self::format_empty(stringify!($e))); - )* - $( - page_data.insert(stringify!($d), Self::format(stringify!($d), $v)); - )* - page_data - } +#[derive(Builder, Clone, Debug, Deserialize, Getters, PartialEq, Serialize)] +pub struct ConfigQuery { + #[builder(default=String::from("https://accounts.google.com/o/oauth2/auth"))] + auth_uri: String, + #[builder(default=String::from("https://www.googleapis.com/oauth2/v1/certs"))] + auth_provider_x509_cert_url: String, + client_email: Option, + client_id: String, + client_secret: String, + client_x509_cert_url: Option, + project_id: String, + redirect_uris: Vec, + #[builder(default=String::from("https://oauth2.googleapis.com/token"))] + token_uri: String, +} - fn format(name: &str, value: &str) -> String { - format!("name=\"{}\" value=\"{}\"", name, value) - } +impl ConfigProperties for ConfigQuery { + fn auth_provider_x509_cert_url(&self) -> &str { + &self.auth_provider_x509_cert_url + } - fn format_empty(name: &str) -> String { - format!("name=\"{}\"", name) - } + fn auth_uri(&self) -> &str { + &self.auth_uri + } - pub(crate) fn to_page_data(&self) -> HashMap<&'static str, String> { - let mut page_data = HashMap::new(); - $( - page_data.insert(stringify!($e), Self::format(stringify!($e), &self.$e)); - )* - $( - page_data.insert(stringify!($d), Self::format(stringify!($d), &self.$d)); - )* - page_data - } - } - } + fn client_email(&self) -> Option<&str> { + self.client_email.as_deref() } -} -make_config_query!( - ConfigQuery, - { project_id, client_id, client_secret }, - { - auth_uri, "https://accounts.google.com/o/oauth2/auth", - auth_provider_x509_cert_url, "https://www.googleapis.com/oauth2/v1/certs", - token_uri, "https://oauth2.googleapis.com/token" + + fn client_id(&self) -> &str { + &self.client_id } -); -impl PersistableConfig for ConfigQuery { - async fn persist(&self, path: &Path) -> Result<(), io::Error> { - debug!("Config path: {:?}", path); - let serialized = serde_yaml::to_string(self).map_err(|e| { - io::Error::new( - io::ErrorKind::InvalidData, - format!("Could not serialize config file due to {:?}.", e), - ) - })?; + fn client_secret(&self) -> &str { + &self.client_secret + } + + fn client_x509_cert_url(&self) -> Option<&str> { + self.client_x509_cert_url.as_deref() + } - fs::write(&path, serialized).await?; + fn project_id(&self) -> &str { + &self.project_id + } - Ok(()) + fn redirect_uris(&self) -> &Vec { + &self.redirect_uris } - async fn read_config(path: &Path) -> Result { - debug!("Config path: {:?}", path); - let slice = fs::read(&path).await.map_err(|e| { - debug!( - "Could not read config file at {} due to {:?}.", - path.display(), - e - ); - e - })?; - debug!("Read result: {:?}", slice); - let config_query = serde_yaml::from_slice(&slice).map_err(|e| { - io::Error::new( - io::ErrorKind::InvalidData, - format!("Could not parse config file due to {:?}.", e), - ) - })?; - debug!("Parse result: {:?}", config_query); - Ok(config_query) + fn token_uri(&self) -> &str { + &self.token_uri } } +impl Config for ConfigQuery {} + impl ConfigQuery { - pub(crate) fn to_application_secret(&self, config: &Config) -> ApplicationSecret { - ApplicationSecretBuilder::default() - .auth_provider_x509_cert_url(Some(self.auth_provider_x509_cert_url().into())) - .auth_uri(self.auth_uri().into()) - .client_email(None) - .client_id(self.client_id().into()) - .client_secret(self.client_secret().into()) - .client_x509_cert_url(None) - .project_id(Some(self.project_id().into())) - .redirect_uris(vec![config.redirect_uri()]) - .token_uri(self.token_uri().into()) - .build() - .unwrap_or_else(|e| { - panic!("Error while building ApplicationSecret: {:?}", e); - }) + pub fn new( + auth_uri: String, + auth_provider_x509_cert_url: String, + client_email: Option, + client_id: String, + client_secret: String, + client_x509_cert_url: Option, + project_id: String, + redirect_uris: Vec, + token_uri: String, + ) -> Self { + Self { + auth_uri, + auth_provider_x509_cert_url, + client_email, + client_id, + client_secret, + client_x509_cert_url, + project_id, + redirect_uris, + token_uri, + } } } diff --git a/cloud_scraper/src/integration/google/auth/delegate.rs b/cloud_scraper/src/integration/google/auth/delegate.rs index 4274b57..cf977b5 100644 --- a/cloud_scraper/src/integration/google/auth/delegate.rs +++ b/cloud_scraper/src/integration/google/auth/delegate.rs @@ -1,21 +1,17 @@ use crate::domain::oauth2::Client; -use derive_builder::Builder; use google_tasks1::common::GetToken; use std::future::Future; use std::pin::Pin; -#[derive(Builder, Clone)] -pub struct Delegate -where - ClientImpl: Client, -{ - client: ClientImpl, +pub struct Delegate { + client: Pin>, } -impl Delegate -where - ClientImpl: Client, -{ +impl Delegate { + pub fn new(client: Pin>) -> Self { + Self { client } + } + async fn get_secret( &self, scopes: &[&str], @@ -28,10 +24,15 @@ where } } -impl GetToken for Delegate -where - ClientImpl: Client, -{ +impl Clone for Delegate { + fn clone(&self) -> Self { + Self { + client: self.client.duplicate(), + } + } +} + +impl GetToken for Delegate { fn get_token<'a>( &'a self, scopes: &'a [&str], diff --git a/cloud_scraper/src/integration/google/auth/mod.rs b/cloud_scraper/src/integration/google/auth/mod.rs index b13448f..dc704f2 100644 --- a/cloud_scraper/src/integration/google/auth/mod.rs +++ b/cloud_scraper/src/integration/google/auth/mod.rs @@ -4,4 +4,3 @@ pub mod web; pub(crate) use config::ConfigQuery; pub use delegate::Delegate; -pub use delegate::DelegateBuilder; diff --git a/cloud_scraper/src/integration/google/auth/web.rs b/cloud_scraper/src/integration/google/auth/web.rs index 0194927..7558077 100644 --- a/cloud_scraper/src/integration/google/auth/web.rs +++ b/cloud_scraper/src/integration/google/auth/web.rs @@ -1,6 +1,6 @@ use crate::core::node_handles::NodeHandles; use crate::domain::node::Manager; -use crate::domain::oauth2::BasicClientImpl; +use crate::domain::oauth2::Config; use crate::domain::oauth2::PersistableConfig; use crate::integration::google::auth::ConfigQuery; use crate::integration::google::Source; @@ -58,7 +58,7 @@ pub fn config_google( } async fn format_response(handles: NodeHandles) -> Result { - let existing_config = Source::::get_auth_config().await.ok(); + let existing_config = Source::get_auth_config().await.ok(); Ok(reply::html( format_config_google_html(handles, &existing_config).await, )) @@ -84,13 +84,13 @@ async fn update_config( ) -> Result { let config = ConfigQuery::from(&form_map); - let path = Source::::config_path() + let path = Source::config_path() .await .map_err(|e| e.into_rejection())?; match config.persist(&path).await { Ok(_) => { let mut sender: Manager = handles.lifecycle_manager().clone(); - match sender.send_read_config::>() { + match sender.send_read_config::() { Ok(_) => { debug!("Google config update sent"); Ok(warp::redirect::found(warp::http::Uri::from_static( @@ -107,6 +107,62 @@ async fn update_config( } } +impl From<&HashMap> for ConfigQuery { + fn from(value: &HashMap) -> Self { + ConfigQuery::new( + value.get("auth_provider_x509_cert_url").unwrap().into(), + value.get("auth_uri").unwrap().into(), + value.get("client_email").map(|s| s.into()), + value.get("client_id").unwrap().into(), + value.get("client_secret").unwrap().into(), + value.get("client_x509_cert_url").map(|s| s.into()), + value.get("project_id").unwrap().into(), + value + .get("redirect_uris") + .map(|s| s.split(',').map(|s| s.to_string()).collect()) + .unwrap_or_default(), + value.get("token_uri").unwrap().into(), + ) + } +} + +trait PageDataMaker { + fn empty_page_data() -> HashMap<&'static str, String>; + fn to_page_data(&self) -> HashMap<&'static str, String>; +} + +impl PageDataMaker for T +where + T: Config, +{ + fn empty_page_data() -> HashMap<&'static str, String> { + HashMap::default() + } + + fn to_page_data(&self) -> HashMap<&'static str, String> { + let mut page_data = HashMap::new(); + page_data.insert( + "auth_provider_x509_cert_url", + self.auth_provider_x509_cert_url().into(), + ); + page_data.insert("auth_uri", self.auth_uri().into()); + if let Some(client_email) = self.client_email() { + page_data.insert("client_email", client_email.into()); + } + page_data.insert("client_id", self.client_id().into()); + page_data.insert("client_secret", self.client_secret().into()); + if let Some(client_x509_cert_url) = self.client_x509_cert_url() { + page_data.insert("client_x509_cert_url", client_x509_cert_url.into()); + } + page_data.insert("project_id", self.project_id().into()); + page_data.insert("token_uri", self.token_uri().into()); + if !self.redirect_uris().is_empty() { + page_data.insert("redirect_uris", self.redirect_uris().join(",")); + } + page_data + } +} + #[cfg(test)] mod tests { use super::*; @@ -127,7 +183,7 @@ mod tests { async fn make_config_file_and_lock<'a>() -> CleanableTestFile<'a> { CleanableTestFile::new( TEST_MUTEX.lock().expect("Could not lock mutex."), - Source::::config_path() + Source::config_path() .await .expect("Could not get config path.") .to_str() @@ -143,7 +199,7 @@ mod tests { } async fn reset() { - let config_path = Source::::config_path().await.unwrap(); + let config_path = Source::config_path().await.unwrap(); let _ = fs::remove_file(&config_path).await; } @@ -278,12 +334,12 @@ mod tests { mod to_application_secret { use super::*; - use crate::domain::{Config, DomainConfig}; + use crate::domain::{Config as ConfigTrait, DomainConfig}; #[test] fn returns_application_secret() { let config = test_config(); - let core_config = Config::with_all_properties( + let core_config = ConfigTrait::with_all_properties( Some(DomainConfig::new("https://localhost")), None, None, @@ -313,7 +369,7 @@ mod tests { #[test] fn preserves_the_url_port() { let config = test_config(); - let core_config = Config::with_all_properties( + let core_config = ConfigTrait::with_all_properties( Some(DomainConfig::new("https://the.domain:8081")), None, None, diff --git a/cloud_scraper/src/integration/google/source.rs b/cloud_scraper/src/integration/google/source.rs index 552b5ec..14316a6 100644 --- a/cloud_scraper/src/integration/google/source.rs +++ b/cloud_scraper/src/integration/google/source.rs @@ -1,17 +1,20 @@ use crate::core::module::State; use crate::domain::module_state::{ModuleState, NamedModule}; use crate::domain::node::{InitReplier, Lifecycle, Manager}; -use crate::domain::oauth2::{extra_parameters, BasicClientImpl, Client, PersistableConfig}; -use crate::integration::google::auth::{ConfigQuery, DelegateBuilder}; +use crate::domain::oauth2::{extra_parameters, Client, Config, PersistableConfig}; +use crate::domain::oauth2::{ApplicationSecret, ExtraParameters}; +use crate::integration::google::auth::{ConfigQuery, Delegate}; use crate::integration::google::tasks::sync; use crate::server::auth::get_token_path; use crate::server::WebEventChannelHandle; use derive_getters::Getters; use log::{debug, error, info, trace}; use std::any::TypeId; +use std::future::Future; use std::io; -use std::marker::PhantomData; +use std::path::Path; use std::path::PathBuf; +use std::pin::Pin; use std::sync::Arc; use std::time::Duration; use tokio::sync::{mpsc, OwnedSemaphorePermit, Semaphore}; @@ -20,37 +23,42 @@ use tokio::{join, task}; use Lifecycle::{Init, ReadConfig, Stop}; #[derive(Clone, Debug, Getters)] -pub struct Source -where - ClientType: Client, -{ - _phantom: PhantomData, +pub struct Source { lifecycle_manager: Manager, web_channel_handle: WebEventChannelHandle, } -impl NamedModule for Source -where - ClientType: Client, -{ +impl NamedModule for Source { fn name() -> &'static str { "google" } } -impl Source -where - ClientType: Client, -{ +impl Source { pub fn new(manager: &Manager, web_channel_handle: &WebEventChannelHandle) -> Self { Self { - _phantom: Default::default(), lifecycle_manager: manager.clone(), web_channel_handle: web_channel_handle.clone(), } } - pub async fn run(&self, google_permit: OwnedSemaphorePermit) { + pub async fn run<'a, T, U, V, W>( + &'a self, + google_permit: OwnedSemaphorePermit, + get_auth_config: U, + client_constructor: fn( + ApplicationSecret, + &ExtraParameters, + &Manager, + &Path, + &WebEventChannelHandle, + ) -> Pin>, + ) where + T: Future> + Send + Sized, + U: Fn(&'a str) -> T + Send + 'static, + V: Config, + W: Client, + { let (load_sender, mut load_receiver) = mpsc::channel(1); let core_config = self.lifecycle_manager.core_config().clone(); let semaphore = Arc::new(Semaphore::new(1)); @@ -84,7 +92,9 @@ where } } - let application_secret = if let Ok(config) = Self::get_auth_config().await { + // get_auth_config taking generics seems to be unsupported by the compiler right now. + // Buttmuppets. I'll just use the concrete type for now. + let application_secret = if let Ok(config) = get_auth_config(Self::name()).await { config.to_application_secret(&core_config) } else { Self::wait_in_loop().await; @@ -97,7 +107,8 @@ where continue; } }; - let client = BasicClientImpl::new( + // This also suffers from the same class of buttmuppetism. + let client = client_constructor( application_secret, &extra_parameters!("access_type" => "offline"), &lifecycle_manager, @@ -109,13 +120,7 @@ where break; } - let delegate = match DelegateBuilder::default().client(client.clone()).build() { - Ok(delegate) => delegate, - Err(e) => { - error!("Error while creating Google authentication delegate: {}", e); - break; - } - }; + let delegate = Delegate::new(client.duplicate()); sync(delegate).await; Self::wait_in_loop().await; } @@ -165,7 +170,7 @@ where event.reply_to_init_with((), "google_source").await } ReadConfig(type_id) => { - if type_id == TypeId::of::>() { + if type_id == TypeId::of::() { send_load!(); } } diff --git a/cloud_scraper/src/integration/google/tasks/sync.rs b/cloud_scraper/src/integration/google/tasks/sync.rs index e3cc0a3..14818ac 100644 --- a/cloud_scraper/src/integration/google/tasks/sync.rs +++ b/cloud_scraper/src/integration/google/tasks/sync.rs @@ -1,12 +1,8 @@ -use crate::domain::oauth2::Client; use crate::integration::google::auth::Delegate; use google_tasks1::{hyper_rustls, TasksHub}; use log::info; -pub(crate) async fn sync(delegate: Delegate) -where - T: Client, -{ +pub(crate) async fn sync(delegate: Delegate) { let client = hyper_util::client::legacy::Client::builder(hyper_util::rt::TokioExecutor::new()) .build( hyper_rustls::HttpsConnectorBuilder::new() diff --git a/cloud_scraper/src/test.rs b/cloud_scraper/src/test.rs index 69ebcd0..5bebd2a 100644 --- a/cloud_scraper/src/test.rs +++ b/cloud_scraper/src/test.rs @@ -12,6 +12,14 @@ pub(crate) mod tests { use std::ops::Deref; use std::sync::{Arc, MutexGuard, Once}; + #[macro_export] + macro_rules! assert_is_send_and_sync { + ($type:ty) => {{ + fn is_send_and_sync() {} + is_send_and_sync::<$type>(); + }}; + } + #[derive(Clone, Debug, PartialEq)] pub(crate) struct LogEntry { args: String, diff --git a/functional_tests/tests/cucumber/google_world.rs b/functional_tests/tests/cucumber/google_world.rs index 7ea9843..bc4a016 100644 --- a/functional_tests/tests/cucumber/google_world.rs +++ b/functional_tests/tests/cucumber/google_world.rs @@ -43,7 +43,7 @@ pub(crate) struct GoogleWorld { #[getter(skip)] manager: Option, #[getter(skip)] - source: Option>>, + source: Option>, source_once: Once, web_channel_handle: WebEventChannelHandle, } @@ -75,7 +75,7 @@ impl GoogleWorld { async fn test<'a>( permit: OwnedSemaphorePermit, semaphore: &'a Arc, - source: &'a Arc>, + source: &'a Arc, ) { drop(permit); source