diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..210d359 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,124 @@ +# AGENTS.md + +This file helps AI coding agents understand the repository structure, build/test conventions, and key architecture decisions for the **Space Weather SOC (SWSOC) AWS Lambda File Sorting Function**. + +## Project Overview + +This is an AWS Lambda function that sorts Space Weather Science Operations Center files into appropriate S3 buckets based on filename parsing. The function: +- Processes S3 `Records` events or performs full incoming-bucket scans +- Routes files to instrument-specific buckets +- Integrates with Slack notifications and AWS Timestream logging +- Operates in **DEVELOPMENT** mode (only sorts `dev_` prefixed files) or **PRODUCTION** mode (sorts all files) + +See [README.rst](README.rst) for detailed documentation and local testing instructions. + +## Essential Commands + +### Testing +```bash +# Run all tests with coverage +pytest --pyargs lambda_function/tests --cov=lambda_function/src --cov-report=html + +# Build Docker container for local Lambda testing +cd lambda_function && docker build -t sdc_aws_sorting_lambda:latest . + +# Run Lambda locally and test with sample event +docker run -p 9000:8080 -v "$(pwd)/tests/test_data:/test_data" sdc_aws_sorting_lambda:latest +curl -XPOST "http://localhost:9000/2015-03-31/functions/function/invocations" -d @tests/test_data/test_padre_event.json +``` + +## Project Structure + +``` +lambda_function/ +├── src/ +│ ├── lambda.py # Handler entry point for Lambda +│ └── file_sorter/ +│ ├── __init__.py +│ └── file_sorter.py # Core FileSorter class and handle_event() logic +├── tests/ +│ ├── conftest.py # Shared pytest fixtures (default_test_mission, use_mission) +│ ├── test_file_sorter.py # Main test suite +│ └── test_data/ # Sample S3 event payloads +├── Dockerfile # Lambda container image +└── requirements.txt # Production dependencies + +lambda_function/src/file_sorter/file_sorter.py imports from swxsoc: + - S3 operations (copy_file_in_s3, check_file_existence_in_target_buckets, etc.) + - Config utilities (get_instrument_bucket, get_incoming_bucket, etc.) + - Slack notifications (get_slack_client, send_pipeline_notification) + - Timestream logging (create_timestream_client_session, log_to_timestream) + - Utility functions (parse_science_filename) +``` + +## Key Technical Details + +**Python Version**: 3.12 (must match AWS Lambda runtime) + +**Environment Variables**: +- `LAMBDA_ENVIRONMENT` (default: `"DEVELOPMENT"`): Controls sorting behavior + - `"DEVELOPMENT"`: Only sorts files with `dev_` prefix + - `"PRODUCTION"`: Sorts all files +- `SWXSOC_MISSION` (default: `"hermes"` in tests): Configures mission-specific behavior + +**Dependencies**: +- `swxsoc` (from git): Core library for S3, Slack, logging, config +- `moto==5.0.15`: Mocks AWS services in tests +- `pytest`, `pytest-astropy`, `pytest-cov`: Testing framework +- `ruff`: Code linting + +**Linting**: Uses Ruff with specific ignores defined in [ruff.toml](ruff.toml) + +## Testing Conventions + +- **Fixtures**: `conftest.py` provides: + - `default_test_mission`: Auto-applied fixture that sets `SWXSOC_MISSION=hermes` for all tests + - `use_mission`: Fixture for tests that need a specific mission configuration + +- **Mocking**: When testing `file_sorter.py`, patch Slack helpers in the module namespace: + ```python + # Patch at the import location in file_sorter.py, not slack_sdk + with patch('file_sorter.file_sorter.send_pipeline_notification'): + ... + ``` + +- **AWS Mocking**: Use `moto` to mock S3 and other AWS services + +## CI/CD Workflows + +See [.github/workflows/](/.github/workflows/) for workflow definitions: +- **testing.yml**: Runs on PR, workflow_dispatch, and daily schedule; runs tests with coverage +- Coverage reports uploaded to Codecov + +## Common Development Tasks + +| Task | Command/Approach | +|------|------------------| +| Run tests | `pytest --pyargs lambda_function/tests --cov=lambda_function/src --cov-report=html` | +| Build Lambda image | `cd lambda_function && docker build -t sdc_aws_sorting_lambda:latest .` | +| Test Lambda locally | See README.rst section "Testing Lambda Locally" | +| Add new test | Place in `lambda_function/tests/test_*.py`; conftest fixtures auto-apply | +| Modify sorting logic | Edit `lambda_function/src/file_sorter/file_sorter.py` | +| Update dependencies | Edit `lambda_function/requirements.txt` (prod) or `requirements.dev.txt` (dev) | + +## Deployment + +The function is deployed as a zip file: +- **Production**: Latest GitHub release +- **Development/Testing**: Latest commit on `main` branch + +## Key Decisions & Patterns + +1. **Environment-based behavior**: The `LAMBDA_ENVIRONMENT` variable controls prod vs. dev sorting to allow testing with real AWS infrastructure without moving production files. + +2. **Modular S3/Slack handling**: All AWS service interactions are abstracted via `swxsoc` library utilities, making this function focused on sorting logic. + +3. **Comprehensive test coverage**: Tests use `moto` to mock AWS services and run without needing actual AWS credentials or S3 buckets. + +4. **Docker for local testing**: Enables testing the exact Lambda runtime environment locally before deployment. + +## Questions or Issues? + +- For Lambda-specific questions, refer to [README.rst](README.rst) +- For swxsoc library details, see the [swxsoc repository](https://github.com/swxsoc/swxsoc) +- For CI/CD, check [.github/workflows/](/.github/workflows/) diff --git a/lambda_function/requirements.txt b/lambda_function/requirements.txt index cc10900..014c45e 100644 --- a/lambda_function/requirements.txt +++ b/lambda_function/requirements.txt @@ -1 +1 @@ -sdc_aws_utils @ git+https://github.com/swxsoc/sdc_aws_utils.git \ No newline at end of file +swxsoc @ git+https://github.com/swxsoc/swxsoc.git@main \ No newline at end of file diff --git a/lambda_function/src/file_sorter/file_sorter.py b/lambda_function/src/file_sorter/file_sorter.py index 2b7987c..65e767d 100755 --- a/lambda_function/src/file_sorter/file_sorter.py +++ b/lambda_function/src/file_sorter/file_sorter.py @@ -8,29 +8,25 @@ from typing import Any from botocore.client import BaseClient -from sdc_aws_utils.aws import ( +from slack_sdk.errors import SlackApiError +from swxsoc import log +from swxsoc.comm.slack import get_slack_client, send_pipeline_notification +from swxsoc.db.timeseries import create_timestream_client_session, log_to_timestream +from swxsoc.io.s3 import ( check_file_existence_in_target_buckets, copy_file_in_s3, create_s3_client_session, create_s3_file_key, - create_timestream_client_session, list_files_in_bucket, - log_to_timestream, object_exists, ) -from sdc_aws_utils.config import ( +from swxsoc.util.config import ( get_all_instrument_buckets, get_incoming_bucket, get_instrument_bucket, ) -from sdc_aws_utils.logging import configure_logger, log -from sdc_aws_utils.slack import get_slack_client, send_pipeline_notification -from slack_sdk.errors import SlackApiError from swxsoc.util.util import parse_science_filename -# Configure logging levels and format -configure_logger() - def handle_event(event: dict[str, Any], context: Any) -> dict[str, int | str]: """ @@ -66,8 +62,8 @@ def handle_event(event: dict[str, Any], context: Any) -> dict[str, int | str]: else: log.info("No records found in event. Checking all files in bucket.") s3_client = create_s3_client_session() - incoming_bucket = get_incoming_bucket(environment) - instrument_buckets = get_all_instrument_buckets(environment) + incoming_bucket = get_incoming_bucket() + instrument_buckets = get_all_instrument_buckets() keys_in_s3 = list_files_in_bucket(s3_client, incoming_bucket) for key in keys_in_s3: try: @@ -190,9 +186,7 @@ def __init__( raise e self.incoming_bucket_name = s3_bucket - self.destination_bucket = get_instrument_bucket( - self.science_file["instrument"], environment - ) + self.destination_bucket = get_instrument_bucket(self.science_file["instrument"]) log.info( f"Sorting from Incoming Bucket: {self.incoming_bucket_name} to Destination Bucket: {self.destination_bucket}" ) @@ -229,7 +223,7 @@ def _sort_file(self): new_file_key = create_s3_file_key(parse_science_filename, path_file.name) except ValueError: log.warning(f"Error parsing file key: {self.file_key}") - return None + return log.info( f"Copying {self.file_key} from {self.incoming_bucket_name}" @@ -240,7 +234,7 @@ def _sort_file(self): log.info( f"Dry Run: Skipping copy of {self.file_key} to {self.destination_bucket}" ) - return None + return # Copy file from source to destination copy_file_in_s3( @@ -270,7 +264,6 @@ def _sort_file(self): new_file_key=new_file_key, source_bucket=self.incoming_bucket_name, destination_bucket=self.destination_bucket, - environment=self.environment, ) log.info( diff --git a/lambda_function/tests/conftest.py b/lambda_function/tests/conftest.py index 5b25d5a..f7b5f9a 100644 --- a/lambda_function/tests/conftest.py +++ b/lambda_function/tests/conftest.py @@ -26,14 +26,11 @@ def default_test_mission(monkeypatch): mission in their example code if they need a specific mission configuration. """ import swxsoc - from sdc_aws_utils.config import _reconfigure_globals # Only set if not already set (allows tests to override) if "SWXSOC_MISSION" not in os.environ: monkeypatch.setenv("SWXSOC_MISSION", "hermes") - swxsoc._reconfigure() - # Re-read module-level globals in config.py so they reflect the new mission - _reconfigure_globals() + swxsoc.reconfigure() @pytest.fixture(scope="function") @@ -73,20 +70,17 @@ def test_all_missions(use_mission): assert swxsoc.config['mission']['mission_name'] == use_mission """ import swxsoc - from sdc_aws_utils.config import _reconfigure_globals mission = request.param if hasattr(request, "param") else "hermes" monkeypatch.setenv("SWXSOC_MISSION", mission) - swxsoc._reconfigure() - _reconfigure_globals() + swxsoc.reconfigure() yield mission # Explicitly reconfigure back to default after test completes # This is necessary because swxsoc.config is module-level state # that persists across tests in the same process # This ensures the config is reset even if monkeypatch cleanup hasn't run yet monkeypatch.setenv("SWXSOC_MISSION", "hermes") - swxsoc._reconfigure() - _reconfigure_globals() + swxsoc.reconfigure() @pytest.fixture(autouse=True, scope="function") diff --git a/lambda_function/tests/test_file_sorter.py b/lambda_function/tests/test_file_sorter.py index f04cfd9..acfdba1 100755 --- a/lambda_function/tests/test_file_sorter.py +++ b/lambda_function/tests/test_file_sorter.py @@ -4,9 +4,10 @@ import boto3 import pytest from moto import mock_aws as moto_mock_aws -from sdc_aws_utils.aws import create_s3_file_key -from sdc_aws_utils.config import get_incoming_bucket, get_instrument_bucket, parser from src.file_sorter import file_sorter +from swxsoc.io.s3 import create_s3_file_key +from swxsoc.util.config import get_incoming_bucket, get_instrument_bucket +from swxsoc.util.util import parse_science_filename TEST_REGION = "us-east-1" ENVIRONMENT = "PRODUCTION" @@ -137,7 +138,7 @@ def create_s3_event(bucket_name, object_key): "bucket": { "name": bucket_name, "ownerIdentity": {"principalId": "EXAMPLE"}, - "arn": "arn:aws:s3:::{}".format(bucket_name), + "arn": f"arn:aws:s3:::{bucket_name}", }, "object": { "key": object_key, @@ -221,8 +222,9 @@ def _get_timestream_names(environment): def setup_case_environment(s3_client, timestream_client, instrument, file_key): """Create case-specific resources for an incoming file and target instrument bucket.""" - incoming_bucket = get_incoming_bucket(ENVIRONMENT) - destination_bucket = get_instrument_bucket(instrument, ENVIRONMENT) + os.environ["LAMBDA_ENVIRONMENT"] = ENVIRONMENT + incoming_bucket = get_incoming_bucket() + destination_bucket = get_instrument_bucket(instrument) setup_environment( s3_client=s3_client, timestream_client=timestream_client, @@ -235,7 +237,7 @@ def setup_case_environment(s3_client, timestream_client, instrument, file_key): def assert_file_sorted(s3_client, destination_bucket, file_key): """Assert that a file was written to the expected destination key.""" - expected_key = create_s3_file_key(parser, Path(file_key).name) + expected_key = create_s3_file_key(parse_science_filename, Path(file_key).name) objects = s3_client.list_objects(Bucket=destination_bucket).get("Contents") assert objects assert objects[0].get("Key") == expected_key @@ -267,7 +269,7 @@ def test_file_sorter(s3_client, timestream_client, use_mission, case): response = file_sorter.handle_event(event=s3_event, context=None) # Successful run should return 200 status code - assert response["statusCode"] == 200 + assert response["statusCode"] == 200, response["body"] assert_file_sorted(s3_client, destination_bucket, case["file_key"]) @@ -347,7 +349,7 @@ def test_file_sorter_empty_trigger(s3_client, timestream_client, use_mission): # Ensure no crash when file already exists in target bucket. s3_client.put_object( Bucket=destination_bucket, - Key=create_s3_file_key(parser, Path(file_key).name), + Key=create_s3_file_key(parse_science_filename, Path(file_key).name), Body=b"test file", ) response = file_sorter.handle_event(event=trigger_event, context=None) @@ -405,8 +407,8 @@ def test_file_sorter_dry_run( def test_file_sorter_missing_timestream(s3_client): """Test handling of missing Timestream client during FileSorter initialization.""" - test_incoming_bucket = get_incoming_bucket("DEVELOPMENT") - test_target_bucket = get_instrument_bucket("spani", "DEVELOPMENT") + test_incoming_bucket = get_incoming_bucket() + test_target_bucket = get_instrument_bucket("spani") test_file_key = "/tests/test_files/hermes_SPANI_l0_2023040-000018_v01.bin" s3_client.create_bucket(Bucket=test_incoming_bucket) s3_client.create_bucket(Bucket=test_target_bucket) diff --git a/requirements.dev.txt b/requirements.dev.txt index 4b4b9a3..e1db3d6 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -1,4 +1,3 @@ -sdc_aws_utils @ git+https://github.com/swxsoc/sdc_aws_utils.git swxsoc @ git+https://github.com/swxsoc/swxsoc.git@main moto==5.0.15 pytest==9.0.3 diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..2e11c79 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,8 @@ +[lint] +ignore = [ + "EXE002", # Executable file but no shebang present + "BLE001", # Do not catch blind exception: `Exception` + "TRY201", # Use `raise` without specifying exception name + "RUF028", # Invalid formatter suppression comment + "SIM115", # Use a context manager for opening files +]