38 improve google login retry behaviour

.github/workflows/ci.yml

@@ -33,6 +33,11 @@ jobs:
           components: llvm-tools-preview
           override: true
           toolchain: stable
+      - name: build
+        uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: --all-features
       - name: test with coverage
         env:
           CARGO_INCREMENTAL: '0'

.gitignore

@@ -99,4 +99,4 @@ lcov.info
 *.json
 config*.yaml
 state/
-*password*
+*password*.yaml

Cargo.toml

@@ -1,55 +1,9 @@
-[package]
-name = "cloud_scraper"
-version = "0.1.0"
-edition = "2021"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-acme2 = "0.5.1"
-async-trait = "0.1.77"
-chrono = { version = "0.4.35", features = ["serde"] }
-clap = { version = "4.0", features = ["derive"] }
-derive-getters = "0.5.0"
-derive_builder = "0.20.1"
-env_logger = "0.11.1"
-google-tasks1 = "6.0.0"
-handlebars = "6.0.0"
-hex = "0.4.3"
-hmac = "0.12.1"
-hyper-util = "0.1.7"
-lazy_static = "1.4.0"
-log = "0.4.20"
-oauth2 = "4.4.2"
-parking_lot = "0.12.1"
-paste = "1.0.15"
-pbkdf2 = "0.12.2"
-rand = "0.8.5"
-rpassword = "7.3.1"
-serde = { version = "1.0.193", features = ["derive"] }
-serde_yaml = { version = "0.9.29", features = [] }
-sha2 = "0.10.8"
-tokio = { version = "1.35.1", features = ["macros", "process", "rt", "rt-multi-thread", "signal", "time"] }
-tokio-stream = "0.1.15"
-url = "2.5.2"
-uuid = { version = "1.6.1", features = ["serde", "v4"] }
-warp = { version = "0.3", features = ["tls"] }
-x509-parser = "0.16.0"
-
-[dev-dependencies]
-cucumber = "0.21.1"
-mockall = "0.13.0"
-once_cell = "1.19.0"
-regex = "1.11.0"
-reqwest = "0.12.9"
-tokio-test = "0.4.3"
+[workspace]
+members = ["cloud_scraper", "functional_tests"]
+resolver = "2"
 
 [profile.release]
 panic = "abort"
 
 [profile.dev]
 panic = "abort"
-
-[[test]]
-name = "cucumber_cli"
-harness = false

cloud scraper.iml

@@ -3,8 +3,10 @@
   <component name="NewModuleRootManager" inherit-compiler-output="true">
     <exclude-output />
     <content url="file://$MODULE_DIR$">
-      <sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
-      <sourceFolder url="file://$MODULE_DIR$/tests" isTestSource="true" />
+      <sourceFolder url="file://$MODULE_DIR$/cloud_scraper/src" isTestSource="false" />
+      <sourceFolder url="file://$MODULE_DIR$/cloud_scraper/tests" isTestSource="true" />
+      <sourceFolder url="file://$MODULE_DIR$/functional_tests/src" isTestSource="false" />
+      <sourceFolder url="file://$MODULE_DIR$/functional_tests/tests" isTestSource="true" />
       <excludeFolder url="file://$MODULE_DIR$/target" />
     </content>
     <orderEntry type="inheritedJdk" />

cloud_scraper/Cargo.toml

@@ -0,0 +1,43 @@
+[package]
+name = "cloud_scraper"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+acme2 = "0.5.1"
+async-trait = "0.1.77"
+chrono = { version = "0.4.35", features = ["serde"] }
+clap = { version = "4.0", features = ["derive"] }
+derive-getters = "0.5.0"
+derive_builder = "0.20.1"
+env_logger = "0.11.1"
+google-tasks1 = "6.0.0"
+handlebars = "6.0.0"
+hex = "0.4.3"
+hmac = "0.12.1"
+hyper-util = "0.1.7"
+lazy_static = "1.4.0"
+log = "0.4.20"
+oauth2 = "4.4.2"
+parking_lot = "0.12.1"
+paste = "1.0.15"
+pbkdf2 = "0.12.2"
+rand = "0.8.5"
+rpassword = "7.3.1"
+serde = { version = "1.0.193", features = ["derive"] }
+serde_yaml = { version = "0.9.29", features = [] }
+sha2 = "0.10.8"
+tokio = { version = "1.35.1", features = ["macros", "process", "rt", "rt-multi-thread", "signal", "time"] }
+tokio-stream = "0.1.15"
+url = "2.5.2"
+uuid = { version = "1.6.1", features = ["serde", "v4"] }
+warp = { version = "0.3", features = ["tls"] }
+x509-parser = "0.16.0"
+
+[dev-dependencies]
+mockall = "0.13.0"
+once_cell = "1.19.0"
+tempfile = "3.15.0"
+tokio-test = "0.4.3"

src/core/engine.rs → cloud_scraper/src/core/engine.rs

@@ -9,13 +9,16 @@ use crate::core::node_handles::NodeHandles;
 use crate::domain::config::Config;
 use crate::domain::mpsc_handle::{one_shot, OneshotMpscSenderHandle};
 use crate::domain::node::{LifecycleChannelHandle, Manager};
+use crate::domain::oauth2::{ApplicationSecret, ExtraParameters};
+use crate::domain::oauth2::{BasicClientImpl, Client};
 use crate::integration::google::Source as GoogleSource;
 use crate::integration::log::Sink as LogSink;
 use crate::integration::stub::Source as StubSource;
 use crate::server::WebServer;
 use core::time::Duration;
 #[cfg(test)]
 use mockall::automock;
+use std::path::Path;
 use std::sync::atomic::AtomicBool;
 use std::sync::atomic::Ordering::SeqCst;
 use tokio::sync::Semaphore;
@@ -66,7 +69,8 @@ where
         let wait_duration = self.manager.core_config().exit_after();
 
         let mut stub_source = StubSource::new(&self.manager);
-        let google_source = GoogleSource::new(&self.manager, self.server.get_web_channel_handle());
+        let google_source: GoogleSource =
+            GoogleSource::new(&self.manager, self.server.get_web_channel_handle());
         let mut log_sink = LogSink::new(&self.manager, &stub_source.get_readonly_channel_handle());
 
         let node_handles = NodeHandles::new(&self.manager, self.server.get_web_channel_handle());
@@ -96,7 +100,15 @@ where
             .expect("Could not acquire semaphore");
         abort_handles.push(join_set.spawn(async move { log_sink.run(log_permit).await }));
         abort_handles.push(join_set.spawn(async move { stub_source.run(stub_permit).await }));
-        abort_handles.push(join_set.spawn(async move { google_source.run(google_permit).await }));
+        abort_handles.push(join_set.spawn(async move {
+            google_source
+                .run(
+                    google_permit,
+                    BasicClientImpl::get_auth_config,
+                    BasicClientImpl::new,
+                )
+                .await
+        }));
 
         let server = self.server.clone();
         abort_handles.push(join_set.spawn(async move {

src/core/module.rs → cloud_scraper/src/core/module.rs

@@ -13,11 +13,15 @@ impl ModuleState for State {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::domain::oauth2::BasicClientImpl;
     use crate::integration::google::Source;
 
     #[tokio::test]
     async fn test_path_for() {
         assert_eq!(State::path(), "state");
-        assert_eq!(State::path_for::<Source>().await.unwrap(), "state/google")
+        assert_eq!(
+            State::path_for::<Source<BasicClientImpl>>().await.unwrap(),
+            "state/google"
+        )
     }
 }

src/domain/mod.rs → cloud_scraper/src/domain/mod.rs

@@ -4,7 +4,7 @@ pub(crate) mod entity;
 pub(crate) mod entity_data;
 pub(crate) mod module_state;
 pub(crate) mod mpsc_handle;
-pub(crate) mod node;
-pub(crate) mod oauth2;
-
+pub mod node;
+pub mod oauth2;
 pub use config::{Config, DomainConfig};
+pub use mpsc_handle::one_shot;

cloud_scraper/src/domain/module_state.rs

@@ -0,0 +1,24 @@
+use async_trait::async_trait;
+use std::path::PathBuf;
+use tokio::fs;
+
+#[async_trait]
+pub trait ModuleState {
+    fn path() -> &'static str;
+    async fn path_for<Module>() -> Result<PathBuf, std::io::Error>
+    where
+        Module: NamedModule,
+    {
+        Self::path_for_name(Module::name()).await
+    }
+
+    async fn path_for_name(name: &str) -> Result<PathBuf, std::io::Error> {
+        let path = PathBuf::from(Self::path()).join(name);
+        fs::create_dir_all(&path).await?;
+        Ok(path)
+    }
+}
+
+pub trait NamedModule {
+    fn name() -> &'static str;
+}

src/domain/oauth2/application_secret.rs → cloud_scraper/src/domain/oauth2/application_secret.rs

@@ -5,7 +5,7 @@ use oauth2::{AuthUrl, ClientId, ClientSecret, RedirectUrl, TokenUrl};
 use serde::{Deserialize, Serialize};
 
 #[derive(Builder, Deserialize, Getters, Serialize)]
-pub(crate) struct ApplicationSecret {
+pub struct ApplicationSecret {
     client_id: String,
     client_secret: String,
     auth_uri: String,
@@ -39,3 +39,13 @@ impl ApplicationSecret {
         .set_redirect_uri(redirect_uri)
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::assert_is_send_and_sync;
+
+    fn send_and_sync() {
+        assert_is_send_and_sync!(ApplicationSecret);
+    }
+}