diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e363806..80e33a8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,12 +31,12 @@ jobs: run: pip install -e .[test] - name: Run tests - run: pytest -k "not example" + run: pytest tests --ignore-glob='tests/test_docs_*.py' -k "not example" env: API_KEY: ${{ secrets.API_KEY }} - name: Run example tests if: matrix.python-version == '3.14' - run: pytest -k "example" + run: pytest tests/example_search_*_test.py env: API_KEY: ${{ secrets.API_KEY }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..17999bd --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,115 @@ +name: Documentation + +on: + push: + branches: [master] + tags: ['v**'] + workflow_dispatch: + +permissions: + contents: read + +jobs: + live-examples: + name: Live documentation examples + if: >- + github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/v') + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + + - uses: astral-sh/setup-uv@v7 + + - name: Install test dependencies + run: | + uv venv + uv pip install -e '.[test]' + + - name: Run live documentation examples + run: >- + .venv/bin/python -m pytest tests/test_docs_examples.py + --require-docs-key -q --junitxml=docs-example-results.xml + env: + API_KEY: ${{ secrets.API_KEY }} + + - uses: actions/upload-artifact@v7 + if: always() + with: + name: docs-example-results + path: docs-example-results.xml + if-no-files-found: warn + + build: + name: Build Sphinx documentation + needs: [live-examples] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + + - uses: astral-sh/setup-uv@v7 + + - name: Install documentation dependencies + run: | + uv venv + uv pip install -e '.[docs,test]' + + - name: Test documentation publishing + run: .venv/bin/python -m pytest tests/test_docs_publishing.py -q + + - name: Check documentation code and test runner without API calls + run: >- + .venv/bin/python -m pytest tests/test_docs_examples.py + tests/test_docs_example_runner.py -k 'not live' -q + + - name: Build HTML documentation + run: .venv/bin/sphinx-build -M html docs docs/_build -W --keep-going + + - name: Build EPUB documentation + run: .venv/bin/sphinx-build -M epub docs docs/_build -W --keep-going + + - uses: actions/upload-artifact@v7 + with: + name: documentation + path: docs/_build/ + if-no-files-found: error + + publish: + name: Publish Read the Docs + needs: [live-examples, build] + if: >- + needs.live-examples.result == 'success' && + needs.build.result == 'success' && + (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && + (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/v')) + runs-on: ubuntu-latest + timeout-minutes: 60 + environment: + name: docs + url: https://serpapi-python.readthedocs.io/ + concurrency: + group: readthedocs-publish + cancel-in-progress: false + queue: max + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + + - name: Sync versions, publish, and wait for RTD + run: python -m scripts.publish_docs + env: + RTD_API_TOKEN: ${{ secrets.RTD_API_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1d6be8f..62943cc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -38,14 +38,14 @@ jobs: run: pip install -e .[test] - name: Run tests - run: pytest -k "not example" + run: pytest --ignore=tests/test_docs_publishing.py -k "not example" env: API_KEY: ${{ secrets.API_KEY }} - - name: Run example tests + - name: Run engine and documentation examples if: matrix.python-version == '3.14' continue-on-error: ${{ inputs.allow_example_test_failures == true }} - run: pytest -k "example" + run: pytest --ignore=tests/test_docs_publishing.py -k "example" --require-docs-key env: API_KEY: ${{ secrets.API_KEY }} diff --git a/.gitignore b/.gitignore index 143977d..f08db2c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ script/ .coverage -docs/build +docs/_build/ dist/ build/ .pytest_cache/ diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 7168b22..31b038d 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -1,32 +1,27 @@ -# .readthedocs.yaml -# Read the Docs configuration file -# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details - -# Required version: 2 -# Set the OS, Python version and other tools you might need build: - os: ubuntu-22.04 + os: ubuntu-24.04 tools: - python: "3.11" - # You can also specify other tool versions: - # nodejs: "19" - # rust: "1.64" - # golang: "1.19" + python: "3.13" + jobs: + pre_build: + - python -m scripts.check_docs_revision + post_build: + - python -m scripts.check_docs_revision -# Build documentation in the "docs/" directory with Sphinx sphinx: configuration: docs/conf.py + fail_on_warning: true -# Optionally build your docs in additional formats such as PDF and ePub formats: - pdf - epub -# Optional but recommended, declare the Python requirements required -# to build your documentation -# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html python: install: - - requirements: docs/requirements.txt + - method: uv + command: pip + path: . + extras: + - docs diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..1e00f06 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,183 @@ +# Contributing + +Bug reports and pull requests are welcome on GitHub. Run commands from the repository root. + +## Development setup + +Use Python 3.13 to work on the package and its documentation. The SDK CI also tests Python 3.8 through 3.14. Create a virtual environment and install the package with its test and documentation dependencies: + +```sh +uv venv --python 3.13 +uv pip install -e '.[test,docs]' +``` + +The editable install uses the Python files in your checkout, so source changes take effect without reinstalling the package. + +Activate the environment on macOS or Linux: + +```sh +source .venv/bin/activate +``` + +On Windows PowerShell: + +```powershell +.\.venv\Scripts\Activate.ps1 +``` + +The commands below use `python` from that environment. + +## Testing + +### Checks that need no API key + +These tests cover response parsing, image uploads with simulated responses, timeouts, exceptions, documentation syntax, and the example runner: + +```sh +python -m pytest tests/test_output_formats.py tests/test_image_upload.py tests/test_timeout.py tests/test_exceptions.py tests/test_docs_examples.py tests/test_docs_example_runner.py -k 'not live' -q +``` + +The file list limits this command to tests that make no external requests. Applying `-k 'not live'` to the entire suite is not enough to run offline: some SDK integration tests do not have `live` in their names. + +### SDK integration tests + +Set `API_KEY` in your shell or editor's run configuration before running integration tests. Documentation examples also accept `SERPAPI_KEY`, but the shared SDK fixtures require `API_KEY`. If you already have `SERPAPI_KEY` set, copy it to `API_KEY` in the same shell: + +```sh +export API_KEY="$SERPAPI_KEY" +``` + +In PowerShell, use `$env:API_KEY = $env:SERPAPI_KEY`. Keep real keys out of source files and commits. + +To match the test selection in the SDK CI workflow: + +```sh +python -m pytest tests --ignore-glob='tests/test_docs_*.py' -k 'not example' -q +``` + +This includes live account, location, search, and pagination checks. To run every discovered test, including the standalone engine examples and documentation examples: + +```sh +python -m pytest -q +``` + +Both commands make real SerpApi requests. The full suite can use more searches than a test run limited to the files you changed. PR CI also runs the standalone engine examples on Python 3.14. Documentation tests run only in the package and documentation release workflows, on `master` or release tags. + +### Testing documentation examples + +The tests discover Python code blocks in the README and all Markdown pages under `docs/`. They run each page as a separate script against SerpApi, including the multiprocessing example's worker processes. The Lens upload examples use the repository's PNG icon as `image.png`. Each page has a five-minute limit, and HTTP errors or JSON responses containing an `error` fail the check even if the example catches the exception. + +With `SERPAPI_KEY` or `API_KEY` set in your environment, run: + +```sh +python -m pytest tests/test_docs_examples.py --require-docs-key -q +``` + +The `--require-docs-key` option fails if neither key is set. Without that option, local runs skip live tests when no key is available. To check syntax and the test runner without making API calls: + +```sh +python -m pytest tests/test_docs_examples.py tests/test_docs_example_runner.py -k 'not live' -q +``` + +Code blocks marked with `docs-test: skip` are checked for syntax but not executed. These cover the old `google-search-results` package and examples that require private proxy or certificate settings, or disable TLS verification. Each marker includes its reason. + +### Testing documentation publishing + +Run the publishing tests locally with: + +```sh +python -m pytest tests/test_docs_publishing.py -q +``` + +These tests use simulated RTD responses to check outgoing HTTP requests, CI event and commit checks, version selection, polling, and cleanup after failures. They need no API keys and do not publish documentation. In CI, they run only in the Documentation workflow on Python 3.13, before the documentation build and publication. The SDK and PyPI release workflows exclude this file. RTD checks the actual checkout with `scripts.check_docs_revision` before and after building, without rerunning the publishing tests. + +### Running a single test or example + +Run one test file while working on that part of the package: + +```sh +python -m pytest tests/test_output_formats.py -q +``` + +To list documentation test IDs without running their examples: + +```sh +python -m pytest tests/test_docs_examples.py --collect-only -q +``` + +For example, run only the Google Lens upload and search page with: + +```sh +python -m pytest 'tests/test_docs_examples.py::test_documentation_examples_live[docs/examples/google-lens-image-upload.md]' --require-docs-key -q +``` + +Use `-x -vv` in place of `-q` to stop at the first failure and show more detail. A syntax failure points to a page and block number. A live failure can come from the example code, an invalid key, exhausted quota, or an upstream API error; inspect the reported engine and error before changing the example. + +When adding a documentation example, use a fenced block labelled `python`. Blocks on the same page execute in order and share variables. Avoid hard-coded dates that expire. Use a `docs-test: skip` marker with a reason only when a block cannot run in the test environment, such as one requiring a user's proxy or certificate. + +## Building the documentation locally + +The documentation uses Sphinx with MyST for Markdown pages and the Read the Docs theme. The development setup above includes its dependencies. Build and serve the HTML site with: + +```sh +python -m sphinx -M html docs docs/_build -W --keep-going +python -m http.server 8000 --bind 127.0.0.1 --directory docs/_build/html +``` + +Open [the local documentation](http://127.0.0.1:8000). After editing a page, rerun the Sphinx build command and refresh your browser. Check the affected page, its code blocks, and the sidebar links. Press Ctrl+C to stop the server. + +Sphinx discovers public APIs with autodoc and reads their docstrings without running the search examples. Sidebar order comes from the toctrees in `docs/index.md`, so documentation filenames do not need numeric prefixes. The `-W` option makes warnings fail the build. + +CI also builds EPUB. Check it locally with: + +```sh +python -m sphinx -M epub docs docs/_build -W --keep-going +``` + +## Building the package + +Build the source distribution and wheel with: + +```sh +uv build +``` + +The files are written to `dist/`. Documentation sources, this guide, and the logo assets are included in the source distribution (`.tar.gz`) so contributors can build the docs from a source release. The wheel contains only the `serpapi` library and its package metadata. Generated docs and documentation dependencies are not part of a normal installation. + +## Documentation publishing + +The [documentation workflow](.github/workflows/docs.yml) runs the live examples before building HTML and EPUB on pushes to `master`, release tags, and manual runs on either ref. It does not run on pull requests. + +After the live examples and documentation build pass, the same workflow publishes to Read the Docs for `master` and `v*` release tags. PR runs never publish. The publishing job uses the `docs` GitHub environment and its `RTD_API_TOKEN` secret. The SerpApi key stays in GitHub as the existing `API_KEY` repository or organization secret. + +The workflow syncs RTD versions, activates the requested version if needed, and waits for the build to finish. `latest` tracks `master`. A release tag has its own version and also updates `stable` when RTD identifies it as the highest stable release. RTD still builds the site from the repository using [.readthedocs.yaml](.readthedocs.yaml); it does not receive the HTML artifact from GitHub. Documentation publishing runs independently of the PyPI release workflow. + +Before requesting a build, the workflow creates a temporary RTD environment variable named `DOCS_CI_REVISION`. It contains the tested commit, the permitted versions, and an expiration time. RTD checks this record against its checkout before and after the Sphinx build. A missing, expired, or different revision stops publication. RTD needs no SerpApi key. The workflow removes the temporary record after publishing, including when a build fails. Publishing jobs run one at a time so they cannot overwrite each other's revision record. + +### Maintainer setup + +Maintainers can configure the existing RTD project and GitHub environment with these steps: + +1. In RTD **Settings**, set **Connected repository** to **No connected repository** and keep **Repository URL** set to `https://github.com/serpapi/serpapi-python.git`. Set **Default branch** to `master` and the configuration file path to `.readthedocs.yaml`. The public repository URL lets RTD clone the source without receiving GitHub push events through the GitHub App. +2. Under RTD **Integrations**, remove incoming GitHub webhook integrations for this project. If an older RTD webhook is also listed in the GitHub repository's **Settings > Webhooks**, disable or remove that webhook. Do not remove integrations for other projects. +3. Under RTD **Automation Rules**, remove rules that activate new versions or change the default version. The workflow handles release activation. Under **Settings > Pull request builds**, turn off **Build pull requests for this project**. GitHub Actions runs the existing SDK and engine example tests on PRs. +4. Under RTD **Environment Variables**, remove `API_KEY` or `SERPAPI_KEY` if you added either for docs tests. Do not add the RTD API token here. The workflow manages `DOCS_CI_REVISION` automatically. +5. Keep `latest` active in **Versions** and use it as the default documentation version during this migration. Existing release tags contain their original docs and build configuration. After the first release containing these changes builds successfully, you can choose `stable` as the default version. +6. Create an RTD API token in your [RTD profile settings](https://app.readthedocs.org/accounts/tokens/), using an account that maintains the `serpapi-python` project. In GitHub, open the repository's **Settings > Environments**, create an environment named `docs`, and add an environment secret named `RTD_API_TOKEN` with that value. Under **Deployment branches and tags**, select **Selected branches and tags** and add a Branch rule for `master` and a Tag rule for `v*`. Leave required reviewers and wait timers disabled if publishing should run without a manual approval. +7. Merge the changes to `master`. In GitHub **Actions > Documentation**, follow **Live documentation examples**, **Build Sphinx documentation**, and **Publish Read the Docs**. The publishing log links to the RTD build. To retry publishing, run the Documentation workflow on `master` or the intended release tag. A manual run on another branch skips the documentation jobs. + +Use GitHub Actions to request builds after this setup. A manual RTD build has no CI revision record and will fail the revision check. If a branch or tag moves between testing and the RTD checkout, rerun the workflow for its current commit. Pushing a `v*` tag also starts the PyPI release workflow, so use an actual package release to test release documentation. + +See the [RTD build API](https://docs.readthedocs.com/platform/stable/api/v3.html#build-triggering), [Git integration settings](https://docs.readthedocs.com/platform/stable/reference/git-integration.html), and [GitHub environment settings](https://docs.github.com/en/actions/how-tos/deploy/configure-and-manage-deployments/manage-environments) for the platform setup details. + +## Publishing a new release + +1. Update the version in `serpapi/__version__.py`. +2. Push a tag for that version. The release pipeline runs automatically: + ```sh + git tag v1.2.3 + git push origin v1.2.3 + ``` + This triggers the [release workflow](.github/workflows/release.yml), which tests, builds, and publishes to PyPI, then smoke-tests the published package. + +> **Required secret:** `API_KEY` (used by the live documentation examples and the published-package smoke test). diff --git a/MANIFEST.in b/MANIFEST.in index 3ba6fe2..4ddc91b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,7 @@ -include README.md HISTORY.md LICENSE -recursive-include tests *.py \ No newline at end of file +include README.md CONTRIBUTING.md HISTORY.md LICENSE +include .readthedocs.yaml +include scripts/check_docs_revision.py scripts/publish_docs.py +recursive-include tests *.py +recursive-include docs *.py *.md *.txt *.css Makefile +include assets/serpapi-logo.svg assets/serpapi-icon.png +prune docs/_build diff --git a/README.md b/README.md index fa646c3..e746dab 100644 --- a/README.md +++ b/README.md @@ -309,11 +309,10 @@ results = client.search({ }) ``` -To search a local image, upload it first and pass its temporary `image_id` to -Google Lens: +To search a local image, place a file named `image.png` in your working directory, upload it, and pass its temporary `image_id` to Google Lens: ```python -upload = client.upload_image("/path/to/image.png") +upload = client.upload_image("image.png") results = client.search({ "engine": "google_lens", "image_id": upload["image_id"], @@ -325,19 +324,6 @@ Uploaded images can be JPG/JPEG, PNG, or WebP files up to 500 KB. The returned - API Documentation: [Google Lens image uploads](https://serpapi.com/google-lens-upload-an-image), [Image API](https://serpapi.com/image-api) -### Search Google Events -```python -import os -import serpapi - -client = serpapi.Client(api_key=os.getenv("API_KEY")) -results = client.search({ - 'engine': 'google_events', - 'q': 'Events in Austin', -}) -``` -- API Documentation: [serpapi.com/google-events-api](https://serpapi.com/google-events-api) - ### Search Google Local Services ```python import os @@ -416,16 +402,4 @@ MIT License. ## Contributing -Bug reports and pull requests are welcome on GitHub. Once dependencies are installed, you can run the tests with `pytest`. - -## Publishing a new release - -1. Update the version in `serpapi/__version__.py`. -2. Push a tag — the release pipeline runs automatically: - ```sh - git tag v1.2.3 - git push origin v1.2.3 - ``` - This triggers the [release workflow](.github/workflows/release.yml), which tests, builds, and publishes to PyPI, then smoke-tests the published package. - -> **Required secret:** `API_KEY` (used in smoke-test live search). +See [CONTRIBUTING.md](CONTRIBUTING.md) for development setup, testing, and publishing instructions. diff --git a/assets/serpapi-icon.png b/assets/serpapi-icon.png new file mode 100644 index 0000000..e3754a2 Binary files /dev/null and b/assets/serpapi-icon.png differ diff --git a/assets/serpapi-logo.svg b/assets/serpapi-logo.svg new file mode 100644 index 0000000..dd3bb9d --- /dev/null +++ b/assets/serpapi-logo.svg @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + SerpApi + diff --git a/docs/Makefile b/docs/Makefile index ed88099..d4bb2cb 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -6,7 +6,7 @@ SPHINXOPTS ?= SPHINXBUILD ?= sphinx-build SOURCEDIR = . -BUILDDIR = build +BUILDDIR = _build # Put it first so that "make" without argument is like "make help". help: diff --git a/docs/_static/custom.css b/docs/_static/custom.css new file mode 100644 index 0000000..f525388 --- /dev/null +++ b/docs/_static/custom.css @@ -0,0 +1,99 @@ +/* Official palette: https://serpapi.com/media */ +:root { + --serpapi-blue: #377fea; + --serpapi-purple: #6937ea; + --serpapi-dark: #313131; + --serpapi-gradient: linear-gradient(47deg, #377fea, #6937ea); + --sd-color-primary: var(--serpapi-purple); + --sd-color-primary-highlight: var(--serpapi-purple); + --sd-color-primary-text: #ffffff; + --sd-color-secondary: var(--serpapi-blue); + --sd-color-tabs-label-active: var(--serpapi-purple); + --sd-color-tabs-label-hover: var(--serpapi-purple); + --sd-color-tabs-underline-active: var(--serpapi-blue); + --sd-color-tabs-underline-hover: var(--serpapi-blue); +} + +.wy-side-nav-search > a img.logo { + width: 210px; + height: auto; +} + +.wy-side-nav-search, +.wy-nav-top { + background-color: #2b2145; + color: #ffffff; + border-bottom: 4px solid var(--serpapi-blue); + border-image: var(--serpapi-gradient) 1; +} + +.wy-nav-top a { + color: #ffffff; +} + +.wy-side-nav-search input[type="text"] { + border-color: var(--serpapi-blue); +} + +.wy-nav-side { + background: var(--serpapi-dark); +} + +.wy-menu-vertical header, +.wy-menu-vertical p.caption { + color: #ffffff; + height: auto; + min-height: 32px; + line-height: 1.5; + padding-top: 0.4em; + padding-bottom: 0.4em; + white-space: normal; +} + +.wy-menu-vertical a:hover { + background-color: var(--serpapi-purple); + color: #ffffff; +} + +.wy-menu-vertical > ul + ul { + margin-top: 12px; +} + +.wy-menu-vertical > ul + ul > li > a { + font-weight: 700; +} + +.wy-menu-vertical li.current > a, +.wy-menu-vertical li.current > a:hover { + color: var(--serpapi-purple); + box-shadow: inset 4px 0 var(--serpapi-blue); +} + +.wy-nav-content { + color: var(--serpapi-dark); +} + +/* Purple stays readable for small text on the light content background. */ +.wy-nav-content a:not(.btn), +.wy-nav-content a:not(.btn):visited, +.rst-content code.literal, +.rst-content a code { + color: var(--serpapi-purple); +} + +.wy-nav-content a:not(.btn):hover { + text-decoration: underline; +} + +html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) > dt { + color: var(--serpapi-purple); + background: rgba(105, 55, 234, 0.06); + border-top-color: var(--serpapi-blue); +} + +.wy-nav-content a:focus-visible, +.wy-menu-vertical a:focus-visible, +.wy-side-nav-search input[type="text"]:focus-visible { + outline: 2px solid var(--serpapi-blue); + outline-offset: 2px; +} diff --git a/docs/_static/serpapi-python.png b/docs/_static/serpapi-python.png deleted file mode 100644 index 4245bb6..0000000 Binary files a/docs/_static/serpapi-python.png and /dev/null differ diff --git a/docs/ai-agents.md b/docs/ai-agents.md new file mode 100644 index 0000000..aaf78c1 --- /dev/null +++ b/docs/ai-agents.md @@ -0,0 +1,33 @@ +--- +title: "AI Agents" +description: "Use SerpApi search tools, Markdown results, and API references in AI agents." +--- + +# AI Agents + +An AI agent can call SerpApi to search the web and use the returned text and links when answering a question. + +## SerpApi Search Tools + +If you use a Python agent SDK, we recommend [SerpApi Search Tools](https://github.com/serpapi/serpapi-search-tools-python). It is a separate package for plugging SerpApi search tools into your agents. + +## Request Markdown Results + +To write your own search tool with `serpapi.Client`, pass `output="md"`. Markdown is plain text with formatting for headings, links, and tables. + +Set your API key as described in [Getting Started](user_guide/getting-started.md#installation), then run: + +```python +import os +import serpapi + +client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"]) +markdown = client.search(engine="google", q="Python release news", output="md") +print(markdown[:500]) +``` + +The example prints the first 500 characters. Pass the full `markdown` string back to your agent as the search tool's result. See [Output Formats](user_guide/output-formats.md) for JSON, Markdown, and HTML examples and the Python values they return. + +## API Parameter Reference + +Give your coding agent [SerpApi's `llms.txt`](https://serpapi.com/llms.txt) when asking it to write search code. This file lists the Markdown documentation for each API. Ask the agent to read the relevant API page for required parameters, optional filters, and response fields. diff --git a/docs/conf.py b/docs/conf.py index 517e0c2..c7089ed 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,69 +1,41 @@ -# Configuration file for the Sphinx documentation builder. -# -# This file only contains a selection of the most common options. For a full -# list see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html +"""Sphinx configuration for local builds and Read the Docs.""" -# -- Path setup -------------------------------------------------------------- - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# import os -import sys - -sys.path.insert(0, os.path.abspath("../..")) import serpapi -# -- Project information ----------------------------------------------------- - -project = "serpapi-python" -copyright = "2023 SerpApi, LLC" +project = "serpapi" +copyright = "2026 SerpApi, LLC" author = "SerpApi, LLC" - -# The full version, including alpha/beta/rc tags release = serpapi.__version__ +version = release + +extensions = ["sphinx.ext.autodoc", "myst_parser", "sphinx_design"] +source_suffix = {".rst": "restructuredtext", ".md": "markdown"} +root_doc = "index" +exclude_patterns = ["_build", "build", "Thumbs.db", ".DS_Store"] +myst_enable_extensions = ["colon_fence"] +myst_heading_anchors = 3 +autodoc_member_order = "bysource" +autodoc_default_options = { + "members": True, + "imported-members": True, + "exclude-members": "HTTPClient, from_http_response", +} - -# -- General configuration --------------------------------------------------- - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = ["sphinx.ext.githubpages", "sphinx.ext.autodoc"] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path. -exclude_patterns = [] - - -# -- Options for HTML output ------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = "alabaster" - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". +html_theme = "sphinx_rtd_theme" html_static_path = ["_static"] - - -# -- Extension configuration ------------------------------------------------- +html_css_files = ["custom.css"] +html_logo = "../assets/serpapi-logo.svg" +html_favicon = "../assets/serpapi-icon.png" html_theme_options = { - "logo": "serpapi-python.png", - "logo_name": "serapi-python", -} - -html_sidebars = { - "**": [ - "about.html", - ] + "logo_only": True, + "style_nav_header_background": "#2b2145", + "collapse_navigation": False, + "sticky_navigation": True, + "navigation_depth": 3, + "titles_only": True, } +html_baseurl = os.environ.get( + "READTHEDOCS_CANONICAL_URL", "https://serpapi-python.readthedocs.io/en/latest/" +) diff --git a/docs/examples/amazon-product-search.md b/docs/examples/amazon-product-search.md new file mode 100644 index 0000000..ef767f4 --- /dev/null +++ b/docs/examples/amazon-product-search.md @@ -0,0 +1,37 @@ +--- +title: "Amazon Product Search" +description: "Search Amazon products and read prices, ratings, delivery details, and product IDs." +--- + +# Amazon Product Search + +Search Amazon for product listings, prices, ratings, delivery details, and sponsored products. Set `k` to the text you want to search for. + +## Search Products + +[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. + +```python +import os +import serpapi + + +client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) + +results = client.search( + engine="amazon", + k="coffee grinder", + amazon_domain="amazon.com", +) + +for product in results.get("organic_results", [])[:5]: + print(product.get("title")) + print(product.get("price"), product.get("rating"), product.get("reviews")) + print(product.get("asin"), product.get("link_clean")) +``` + +## Read the Results + +Read product listings from `organic_results`. The `asin` field is Amazon's product ID. Useful fields include `title`, `asin`, `price`, `extracted_price`, `rating`, `reviews`, `thumbnail`, `delivery`, `prime`, `link_clean`, and `serpapi_link`. Some searches also return `product_ads`, `featured_products`, `video_results`, `filters`, and `related_searches`. + +For country settings, sorting, product categories, and filters, see the [Amazon Search API documentation](https://serpapi.com/amazon-search-api). Use the [SerpApi Playground](https://serpapi.com/playground) to inspect the response fields for your product category. diff --git a/docs/examples/baidu-search.md b/docs/examples/baidu-search.md new file mode 100644 index 0000000..8b07f7f --- /dev/null +++ b/docs/examples/baidu-search.md @@ -0,0 +1,35 @@ +--- +title: "Baidu Search" +description: "Search Baidu and read web results for research about China." +--- + +# Baidu Search + +Search Baidu for web pages, for example to research a market in China or check where a website appears in search results. + +## Search Baidu + +[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. + +```python +import os +import serpapi + + +client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) + +results = client.search( + engine="baidu", + q="coffee", +) + +for result in results.get("organic_results", [])[:5]: + print(result.get("position"), result.get("title")) + print(result.get("link")) +``` + +## Read the Results + +Read web listings from `organic_results`. The response may contain other sections depending on the query. Inspect a sample response to see which fields are available before choosing what to store. + +See the [Baidu Search API documentation](https://serpapi.com/baidu-search-api) for supported filters. diff --git a/docs/examples/bing-search.md b/docs/examples/bing-search.md new file mode 100644 index 0000000..14f8302 --- /dev/null +++ b/docs/examples/bing-search.md @@ -0,0 +1,40 @@ +--- +title: "Bing Search" +description: "Run a Bing web search and read result titles, links, and positions." +--- + +# Bing Search + +Search Bing to read its web results or compare them with Google results. The example below searches for coffee from Austin, Texas. You can add language, location, and pagination parameters to the same request. + +## Search Bing + +[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. + +```python +import os +import serpapi + + +client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) + +results = client.search( + engine="bing", + q="coffee", + location="Austin, Texas", +) + +for result in results.get("organic_results", [])[:5]: + print(result.get("position"), result.get("title")) + print(result.get("link")) +``` + +## Read the Results + +Read web listings from `organic_results`. Each listing can include a `title`, `link`, `snippet`, and `position`. To see which other sections the response contains, print its keys: + +```python +print(results.keys()) +``` + +For all supported Bing parameters, see the [Bing Search API documentation](https://serpapi.com/bing-search-api). To try different parameters in your browser, use the [SerpApi Playground](https://serpapi.com/playground). diff --git a/docs/examples/categories/ai-answers.md b/docs/examples/categories/ai-answers.md new file mode 100644 index 0000000..55e332e --- /dev/null +++ b/docs/examples/categories/ai-answers.md @@ -0,0 +1,7 @@ +# AI Answers + +```{toctree} +:maxdepth: 1 + +../google-ai-overview +``` diff --git a/docs/examples/categories/finance-trends-and-jobs.md b/docs/examples/categories/finance-trends-and-jobs.md new file mode 100644 index 0000000..40e4814 --- /dev/null +++ b/docs/examples/categories/finance-trends-and-jobs.md @@ -0,0 +1,10 @@ +# Finance, Trends, and Jobs + +```{toctree} +:maxdepth: 1 + +../google-finance-market-data +../google-trends-demand +../google-news-monitoring +../google-jobs-listings +``` diff --git a/docs/examples/categories/local-and-maps.md b/docs/examples/categories/local-and-maps.md new file mode 100644 index 0000000..b701310 --- /dev/null +++ b/docs/examples/categories/local-and-maps.md @@ -0,0 +1,8 @@ +# Local and Maps + +```{toctree} +:maxdepth: 1 + +../google-maps-local-business +../google-local-services +``` diff --git a/docs/examples/categories/media-apps-and-research.md b/docs/examples/categories/media-apps-and-research.md new file mode 100644 index 0000000..aff4cf0 --- /dev/null +++ b/docs/examples/categories/media-apps-and-research.md @@ -0,0 +1,11 @@ +# Media, Apps, and Research + +```{toctree} +:maxdepth: 1 + +../youtube-video-search +../google-images-search +../google-lens-image-upload +../google-scholar-research +../google-play-store-apps +``` diff --git a/docs/examples/categories/search-engines.md b/docs/examples/categories/search-engines.md new file mode 100644 index 0000000..8f2e931 --- /dev/null +++ b/docs/examples/categories/search-engines.md @@ -0,0 +1,10 @@ +# Search Engines + +```{toctree} +:maxdepth: 1 + +../google-across-countries +../bing-search +../duckduckgo-search +../baidu-search +``` diff --git a/docs/examples/categories/shopping-and-marketplaces.md b/docs/examples/categories/shopping-and-marketplaces.md new file mode 100644 index 0000000..cad4a13 --- /dev/null +++ b/docs/examples/categories/shopping-and-marketplaces.md @@ -0,0 +1,12 @@ +# Shopping and Marketplaces + +```{toctree} +:maxdepth: 1 + +../google-shopping-products +../google-shopping-price-monitoring +../amazon-product-search +../walmart-product-search +../ebay-product-listings +../home-depot-product-search +``` diff --git a/docs/examples/categories/travel-and-hospitality.md b/docs/examples/categories/travel-and-hospitality.md new file mode 100644 index 0000000..d51a726 --- /dev/null +++ b/docs/examples/categories/travel-and-hospitality.md @@ -0,0 +1,8 @@ +# Travel and Hospitality + +```{toctree} +:maxdepth: 1 + +../google-flights-travel +../tripadvisor-travel-search +``` diff --git a/docs/examples/duckduckgo-search.md b/docs/examples/duckduckgo-search.md new file mode 100644 index 0000000..6527419 --- /dev/null +++ b/docs/examples/duckduckgo-search.md @@ -0,0 +1,35 @@ +--- +title: "DuckDuckGo Search" +description: "Run a DuckDuckGo search and read web result fields." +--- + +# DuckDuckGo Search + +Search DuckDuckGo to read its web results or compare them with Google and Bing. You can narrow the search with region and time filters. + +## Search DuckDuckGo + +[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. + +```python +import os +import serpapi + + +client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) + +results = client.search( + engine="duckduckgo", + q="coffee", +) + +for result in results.get("organic_results", [])[:5]: + print(result.get("position"), result.get("title")) + print(result.get("link")) +``` + +## Read the Results + +Read web listings from `organic_results`. Each listing can include `title`, `link`, `snippet`, and `position`. Inspect the response for other sections returned by your query. + +See the [DuckDuckGo Search API documentation](https://serpapi.com/duckduckgo-search-api) for region, date, and safe-search options. diff --git a/docs/examples/ebay-product-listings.md b/docs/examples/ebay-product-listings.md new file mode 100644 index 0000000..49bd417 --- /dev/null +++ b/docs/examples/ebay-product-listings.md @@ -0,0 +1,36 @@ +--- +title: "eBay Product Listings" +description: "Search eBay listings and read product, price, and seller fields." +--- + +# eBay Product Listings + +Search eBay for listings with prices, item conditions, and seller information. Set `_nkw` to your search text. + +## Search eBay + +[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. + +```python +import os +import serpapi + + +client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) + +results = client.search( + engine="ebay", + _nkw="coffee", +) + +for listing in results.get("organic_results", [])[:5]: + print(listing.get("title")) + print(listing.get("price"), listing.get("condition")) + print(listing.get("link")) +``` + +## Read the Results + +Read the listings from `organic_results`. Useful fields include `title`, `price`, `condition`, `shipping`, `location`, `seller`, `thumbnail`, and `link`. + +See the [eBay Search API documentation](https://serpapi.com/ebay-search-api) for filters, sorting, and pagination. diff --git a/docs/examples/google-across-countries.md b/docs/examples/google-across-countries.md new file mode 100644 index 0000000..a94f184 --- /dev/null +++ b/docs/examples/google-across-countries.md @@ -0,0 +1,73 @@ +--- +title: "Google Across Countries" +description: "Run the same Google Search across several countries and locations." +--- + +# Google Across Countries + +Compare Google results by setting the country (`gl`), interface language (`hl`), and search location (`location`). Use `google_domain` if you need a particular Google domain. + +## Compare Several Countries + +[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. + +```python +import os +import serpapi + + +client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) + +markets = [ + {"name": "United States", "gl": "us", "hl": "en", "location": "Austin, Texas"}, + {"name": "United Kingdom", "gl": "gb", "hl": "en", "location": "London, England"}, + {"name": "France", "gl": "fr", "hl": "fr", "location": "Paris, France"}, + {"name": "Germany", "gl": "de", "hl": "de", "location": "Berlin, Germany"}, + {"name": "India", "gl": "in", "hl": "en", "location": "Mumbai, Maharashtra"}, +] + +for market in markets: + results = client.search( + engine="google", + q="best coffee beans", + location=market["location"], + gl=market["gl"], + hl=market["hl"], + ) + + organic = results.get("organic_results", []) + first = organic[0] if organic else {} + + print(market["name"]) + print(first.get("title")) + print(first.get("link")) + print() +``` + +## Use a Google Domain + +Set `google_domain="google.co.in"` to search through Google's Indian domain: + +```python +results = client.search( + engine="google", + q="best coffee beans", + google_domain="google.co.in", + gl="in", + hl="en", + location="Mumbai, Maharashtra", +) +``` + +## Finding Valid Locations + +Call `client.locations()` to find location names accepted by SerpApi: + +```python +locations = client.locations(q="Mumbai", limit=5) + +for location in locations: + print(location.get("canonical_name")) +``` + +See [Account and Locations](../user_guide/account-and-locations.md) for details on choosing a location. Use the [SerpApi Playground](https://serpapi.com/playground) to test location values with the rest of your search parameters. diff --git a/docs/examples/google-ai-overview.md b/docs/examples/google-ai-overview.md new file mode 100644 index 0000000..2539d54 --- /dev/null +++ b/docs/examples/google-ai-overview.md @@ -0,0 +1,46 @@ +--- +title: "Google AI Overview" +description: "Read a Google AI Overview answer or retrieve it with a page token." +--- + +# Google AI Overview + +Some Google searches include an AI Overview answer. If the response contains `ai_overview.page_token`, use that token in a second request to retrieve the answer. Tokens expire quickly, so make the second request as soon as you receive one. + +## Search and Fetch AI Overview + +[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. + +```python +import os +import serpapi + + +client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) + +search = client.search( + engine="google", + q="how do noise cancelling headphones work", + location="Austin, Texas", + gl="us", + hl="en", +) + +ai_overview = search.get("ai_overview", {}) + +if ai_overview.get("page_token"): + response = client.search( + engine="google_ai_overview", + page_token=ai_overview["page_token"], + ) + ai_overview = response.get("ai_overview", {}) + +for block in ai_overview.get("text_blocks", [])[:3]: + print(block.get("snippet") or block.get("text")) +``` + +## Read the Results + +The `ai_overview` object can contain `text_blocks` with the answer and `references` with its cited sources. If the first Google Search response already includes the AI Overview content, you can read it directly without the second request. + +See the [Google Search API AI Overview docs](https://serpapi.com/search-api#api-examples-results-for-ai-overview) and the [Google AI Overview API documentation](https://serpapi.com/google-ai-overview-api). Use the [SerpApi Playground](https://serpapi.com/playground) to find queries that currently return AI Overview data. diff --git a/docs/examples/google-finance-market-data.md b/docs/examples/google-finance-market-data.md new file mode 100644 index 0000000..ef9140f --- /dev/null +++ b/docs/examples/google-finance-market-data.md @@ -0,0 +1,40 @@ +--- +title: "Google Finance Market Data" +description: "Read prices, charts, and related news from Google Finance." +--- + +# Google Finance Market Data + +Look up stocks, indexes, mutual funds, currencies, and futures on Google Finance. You can use the returned prices and charts in a watchlist or dashboard. + +## Fetch a Quote + +[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. + +```python +import os +import serpapi + + +client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) + +results = client.search( + engine="google_finance", + q="GOOGL:NASDAQ", + window="1M", + hl="en", +) + +summary = results.get("summary", {}) +knowledge = results.get("knowledge_graph", {}) + +print(summary.get("title") or knowledge.get("title")) +print(summary.get("price") or knowledge.get("price")) +print("Graph points:", len(results.get("graph", []))) +``` + +## Read the Results + +Read the name and price from `summary` or `knowledge_graph`, depending on the response. The `graph` list contains values over time, and `news_results` contains related headlines when available. Set `window` to choose the chart's time range. + +See the [Google Finance API documentation](https://serpapi.com/google-finance-api) for supported `q` formats and time windows. You can test symbols in the [SerpApi Playground](https://serpapi.com/playground). diff --git a/docs/examples/google-flights-travel.md b/docs/examples/google-flights-travel.md new file mode 100644 index 0000000..01d4afa --- /dev/null +++ b/docs/examples/google-flights-travel.md @@ -0,0 +1,45 @@ +--- +title: "Google Flights Travel" +description: "Search for round-trip flights with Google Flights." +--- + +# Google Flights Travel + +Search Google Flights to compare routes, prices, and itineraries. This example searches for a one-week trip starting 30 days from today. `AUS` is Austin and `LAX` is Los Angeles; use the airport codes for your own route. + +## Search Round-Trip Flights + +[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. + +```python +import os +import serpapi +from datetime import date, timedelta + + +client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=30) + +results = client.search( + engine="google_flights", + departure_id="AUS", + arrival_id="LAX", + outbound_date=(date.today() + timedelta(days=30)).isoformat(), + return_date=(date.today() + timedelta(days=37)).isoformat(), + currency="USD", + hl="en", + gl="us", +) + +flights = results.get("best_flights") or results.get("other_flights", []) + +for itinerary in flights[:3]: + print("Total price:", itinerary.get("price")) + for flight in itinerary.get("flights", []): + print(flight.get("departure_airport", {}).get("id"), "->", flight.get("arrival_airport", {}).get("id")) +``` + +## Read the Results + +The example reads `best_flights` if it contains results, then tries `other_flights`. Useful fields include `price`, `total_duration`, `carbon_emissions`, and the `flights` list inside each itinerary for airlines, airports, and departure and arrival times. + +For trip type, cabin, date, currency, and airport parameters, see the [Google Flights API documentation](https://serpapi.com/google-flights-api). The [SerpApi Playground](https://serpapi.com/playground) lets you test a route before copying the parameters into Python. diff --git a/docs/examples/google-images-search.md b/docs/examples/google-images-search.md new file mode 100644 index 0000000..f9a8e1d --- /dev/null +++ b/docs/examples/google-images-search.md @@ -0,0 +1,37 @@ +--- +title: "Google Images Search" +description: "Search Google Images and read image titles, sources, and URLs." +--- + +# Google Images Search + +Search Google Images for thumbnails, source pages, and links to the original images. + +## Search Images + +[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. + +```python +import os +import serpapi + + +client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) + +results = client.search( + engine="google_images", + tbm="isch", + q="coffee", +) + +for image in results.get("images_results", [])[:5]: + print(image.get("title")) + print(image.get("source")) + print(image.get("original")) +``` + +## Read the Results + +Read the images from `images_results`. Useful fields include `title`, `source`, `link`, `thumbnail`, `original`, `original_width`, and `original_height`. + +See the [Google Images API documentation](https://serpapi.com/google-images-api) for image filters and pagination. diff --git a/docs/examples/google-jobs-listings.md b/docs/examples/google-jobs-listings.md new file mode 100644 index 0000000..f2ab6d5 --- /dev/null +++ b/docs/examples/google-jobs-listings.md @@ -0,0 +1,39 @@ +--- +title: "Google Jobs Listings" +description: "Search job listings by role and location." +--- + +# Google Jobs Listings + +Search Google Jobs for open roles in a location. Results include job titles, employers, and listing sources. You can use them to track hiring or research advertised salaries when salary details are available. + +## Search Jobs + +[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. + +```python +import os +import serpapi + + +client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) + +results = client.search( + engine="google_jobs", + q="software engineer", + location="Austin, Texas", + hl="en", + gl="us", +) + +for job in results.get("jobs_results", [])[:5]: + print(job.get("title")) + print(job.get("company_name"), "-", job.get("location")) + print(job.get("via")) +``` + +## Read the Results + +Read job listings from `jobs_results`. Typical fields include `title`, `company_name`, `location`, `via`, `description`, `detected_extensions`, and `related_links`. Use the returned links and identifiers to find further details about a listing. + +See the [Google Jobs API documentation](https://serpapi.com/google-jobs-api) for language, location, and pagination parameters. Use the [SerpApi Playground](https://serpapi.com/playground) to try different roles and locations. diff --git a/docs/examples/google-lens-image-upload.md b/docs/examples/google-lens-image-upload.md new file mode 100644 index 0000000..0f1dcf6 --- /dev/null +++ b/docs/examples/google-lens-image-upload.md @@ -0,0 +1,44 @@ +--- +title: "Google Lens Image Upload" +description: "Upload a local image to SerpApi and search for visual matches with Google Lens." +--- + +# Google Lens Image Upload + +Search Google Lens with an image on your computer. First upload the file with `client.upload_image()`, then pass the returned `image_id` to `client.search()`. + +## Upload and Search + +[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. Place an image named `image.png` in the directory where you run the script, or replace `image.png` with the path to your file. + +Uploads can be JPG/JPEG, PNG, or WebP files up to 500 KB. The returned image ID expires after ten minutes, so search soon after uploading. See the [Image API documentation](https://serpapi.com/image-api) for upload requirements. + +```python +import os +import serpapi + + +client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=30) + +upload = client.upload_image("image.png") + +results = client.search( + engine="google_lens", + image_id=upload["image_id"], + type="visual_matches", + hl="en", +) + +for match in results.get("visual_matches", [])[:5]: + print(match.get("title")) + print(match.get("source")) + print(match.get("link")) +``` + +`upload_image()` returns a dictionary containing `image_id`. The second call uses that ID to search Google Lens. You can also pass a file opened in binary mode to `upload_image()`. See {py:meth}`serpapi.Client.upload_image` in the API reference. + +## Read the Results + +Read matching images and pages from `visual_matches`. Each match can include a `title`, `source`, `link`, and `thumbnail`. The example prints up to five matches and prints nothing if that list is empty. + +Set `type="products"` to look for products, or `type="exact_matches"` to find pages containing the same image. See the [Google Lens API documentation](https://serpapi.com/google-lens-api) for search types and other parameters. diff --git a/docs/examples/google-local-services.md b/docs/examples/google-local-services.md new file mode 100644 index 0000000..c82a1c6 --- /dev/null +++ b/docs/examples/google-local-services.md @@ -0,0 +1,37 @@ +--- +title: "Google Local Services" +description: "Read Google Local Services advertisements for a service category." +--- + +# Google Local Services + +Search Google Local Services for advertisements from local providers, such as electricians. Results include ratings and phone numbers when available. Use the Google Maps API for general place listings. + +## Search Local Services + +[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. + +```python +import os +import serpapi + + +client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) + +results = client.search( + engine="google_local_services", + q="electrician", + data_cid="6745062158417646970", +) + +for ad in results.get("local_ads", [])[:5]: + print(ad.get("title")) + print(ad.get("rating"), ad.get("reviews")) + print(ad.get("phone")) +``` + +## Read the Results + +Read the advertisements from `local_ads`. Useful fields include `title`, `rating`, `reviews`, `phone`, `service_area`, `years_in_business`, and `link` when present. + +See the [Google Local Services API documentation](https://serpapi.com/google-local-services-api) for query and provider detail options. diff --git a/docs/examples/google-maps-local-business.md b/docs/examples/google-maps-local-business.md new file mode 100644 index 0000000..b227ea7 --- /dev/null +++ b/docs/examples/google-maps-local-business.md @@ -0,0 +1,38 @@ +--- +title: "Google Maps Local Businesses" +description: "Find local businesses with Google Maps." +--- + +# Google Maps Local Businesses + +Search Google Maps for nearby businesses, for example to build a store locator, find potential customers, or check local search rankings. Set `q` to the business type or name. The `ll` value specifies the map's latitude, longitude, and zoom level. + +## Search for Local Businesses + +[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. + +```python +import os +import serpapi + + +client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) + +results = client.search( + engine="google_maps", + q="coffee shops", + ll="@30.2672,-97.7431,14z", + type="search", +) + +for place in results.get("local_results", [])[:5]: + print(place.get("title")) + print(place.get("rating"), place.get("reviews")) + print(place.get("address")) +``` + +## Read the Results + +Read business listings from `local_results`. Common fields include `title`, `rating`, `reviews`, `address`, `phone`, `website`, and `gps_coordinates`. Some responses also include `place_results` when the query matches a specific place. + +For the complete parameter list, use the [Google Maps API documentation](https://serpapi.com/google-maps-api). Use the [SerpApi Playground](https://serpapi.com/playground) to choose an `ll` value for the area you want to search. diff --git a/docs/examples/google-news-monitoring.md b/docs/examples/google-news-monitoring.md new file mode 100644 index 0000000..63c1dfd --- /dev/null +++ b/docs/examples/google-news-monitoring.md @@ -0,0 +1,38 @@ +--- +title: "Google News Monitoring" +description: "Collect recent headlines for a topic with Google News." +--- + +# Google News Monitoring + +Search Google News for articles about a topic. You can collect headlines over time or use new matches to trigger alerts. + +## Search News + +[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. + +```python +import os +import serpapi + + +client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) + +results = client.search( + engine="google_news", + q="artificial intelligence", + gl="us", + hl="en", +) + +for item in results.get("news_results", [])[:5]: + print(item.get("title")) + print(item.get("source", {}).get("name") or item.get("source")) + print(item.get("link")) +``` + +## Read the Results + +Read articles from `news_results`. Save `title`, `link`, `source`, `date`, `snippet`, and `thumbnail` when present. Response fields vary by query, so inspect a sample response before choosing which fields to store. + +See the [Google News API documentation](https://serpapi.com/google-news-api) and try your topic in the [SerpApi Playground](https://serpapi.com/playground). diff --git a/docs/examples/google-play-store-apps.md b/docs/examples/google-play-store-apps.md new file mode 100644 index 0000000..f60fc35 --- /dev/null +++ b/docs/examples/google-play-store-apps.md @@ -0,0 +1,38 @@ +--- +title: "Google Play Store Apps" +description: "Search Google Play for apps and read their titles, ratings, and links." +--- + +# Google Play Store Apps + +Search Google Play for apps and read their ratings, descriptions, and store links. + +## Search Apps + +[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. + +```python +import os +import serpapi + + +client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) + +results = client.search( + engine="google_play", + q="kite", + store="apps", + max_results="2", +) + +for app in results.get("organic_results", [])[:5]: + print(app.get("title")) + print(app.get("rating"), app.get("downloads")) + print(app.get("link")) +``` + +## Read the Results + +Read app listings from `organic_results`. Useful fields include `title`, `link`, `rating`, `downloads`, `description`, `thumbnail`, and app identifiers. + +See the [Google Play Store API documentation](https://serpapi.com/google-play-api) for store, device, and localization options. diff --git a/docs/examples/google-scholar-research.md b/docs/examples/google-scholar-research.md new file mode 100644 index 0000000..22b78e0 --- /dev/null +++ b/docs/examples/google-scholar-research.md @@ -0,0 +1,36 @@ +--- +title: "Google Scholar Research" +description: "Search Google Scholar for publications and citation counts." +--- + +# Google Scholar Research + +Search Google Scholar for publications, citation counts, and author links. Repeat a query to check for new research on a topic. + +## Search Scholar + +[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. + +```python +import os +import serpapi + + +client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) + +results = client.search( + engine="google_scholar", + q="coffee", +) + +for result in results.get("organic_results", [])[:5]: + print(result.get("title")) + print(result.get("publication_info", {}).get("summary")) + print(result.get("inline_links", {}).get("cited_by", {}).get("total")) +``` + +## Read the Results + +Read publications from `organic_results`. Useful fields include `title`, `link`, `publication_info`, `snippet`, `resources`, and `inline_links.cited_by`. + +See the [Google Scholar API documentation](https://serpapi.com/google-scholar-api) for author, citation, and date filters. diff --git a/docs/examples/google-shopping-price-monitoring.md b/docs/examples/google-shopping-price-monitoring.md new file mode 100644 index 0000000..71bd727 --- /dev/null +++ b/docs/examples/google-shopping-price-monitoring.md @@ -0,0 +1,43 @@ +--- +title: "Google Shopping Price Monitoring" +description: "Filter Google Shopping listings by price and read selected product fields." +--- + +# Google Shopping Price Monitoring + +Filter Google Shopping listings by price to compare offers from different merchants. The example searches for espresso machines priced between 100 and 800 in the search's currency. + +## Search Filtered Products + +[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. + +```python +import os +import serpapi + + +client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) + +results = client.search( + engine="google_shopping", + q="espresso machine", + location="Austin, Texas", + gl="us", + hl="en", + min_price=100, + max_price=800, + sort_by=1, + json_restrictor="shopping_results[].{title, price, source, rating, reviews, link}", +) + +for product in results.get("shopping_results", [])[:5]: + print(product.get("title")) + print(product.get("price"), product.get("source")) + print(product.get("rating"), product.get("reviews")) +``` + +## Read the Results + +Read product listings from `shopping_results`. Useful fields include `title`, `product_id`, `price`, `extracted_price`, `source`, `rating`, `reviews`, `thumbnail`, `delivery`, `product_link`, and `serpapi_immersive_product_api`. + +Use `min_price`, `max_price`, `sort_by`, `free_shipping`, and `on_sale` to filter and order offers. To fetch more pages, use `serpapi_pagination.next` when it is present. The example limits response fields with `json_restrictor`. Include `serpapi_pagination` in that selector if you need pagination. See the [Google Shopping API documentation](https://serpapi.com/google-shopping-api) for the full parameter set. diff --git a/docs/examples/google-shopping-products.md b/docs/examples/google-shopping-products.md new file mode 100644 index 0000000..061a228 --- /dev/null +++ b/docs/examples/google-shopping-products.md @@ -0,0 +1,39 @@ +--- +title: "Google Shopping Products" +description: "Search Google Shopping and read product listings." +--- + +# Google Shopping Products + +Search Google Shopping for products, prices, merchants, and product links. Use the results to compare prices, research products, or add merchant information to a product catalog. + +## Search Products + +[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. + +```python +import os +import serpapi + + +client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) + +results = client.search( + engine="google_shopping", + q="espresso machine", + location="Austin, Texas", + gl="us", + hl="en", +) + +for product in results.get("shopping_results", [])[:5]: + print(product.get("title")) + print(product.get("price"), product.get("source")) + print(product.get("link")) +``` + +## Read the Results + +Read product listings from `shopping_results`. Save `title`, `price`, `source`, `rating`, `reviews`, `thumbnail`, and `link` if present. Check the API documentation for supported filters and sorting options before adding them to your request. + +See the [Google Shopping API documentation](https://serpapi.com/google-shopping-api) and experiment in the [SerpApi Playground](https://serpapi.com/playground). diff --git a/docs/examples/google-trends-demand.md b/docs/examples/google-trends-demand.md new file mode 100644 index 0000000..5851d4f --- /dev/null +++ b/docs/examples/google-trends-demand.md @@ -0,0 +1,41 @@ +--- +title: "Google Trends Demand" +description: "Compare search interest over time with Google Trends." +--- + +# Google Trends Demand + +Use Google Trends to compare search interest over time and between regions. You can look for seasonal patterns when planning content, researching products, or choosing a launch date. + +## Interest Over Time + +[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. + +```python +import os +import serpapi + + +client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) + +results = client.search( + engine="google_trends", + q="electric bikes", + date="today 12-m", + geo="US", + data_type="TIMESERIES", + tz="420", +) + +timeline = results.get("interest_over_time", {}).get("timeline_data", []) + +for point in timeline[-5:]: + value = point.get("values", [{}])[0].get("extracted_value") + print(point.get("date"), value) +``` + +## Read the Results + +Read `interest_over_time.timeline_data` for search interest over time. Set `data_type="GEO_MAP_0"` to compare regions, or use `RELATED_QUERIES` or `RELATED_TOPICS` to find related searches. + +See the [Google Trends API documentation](https://serpapi.com/google-trends-api) for `data_type`, `geo`, `date`, and `tz` options. The [SerpApi Playground](https://serpapi.com/playground) helps confirm that a trend query returns the chart you expect. diff --git a/docs/examples/home-depot-product-search.md b/docs/examples/home-depot-product-search.md new file mode 100644 index 0000000..51a897f --- /dev/null +++ b/docs/examples/home-depot-product-search.md @@ -0,0 +1,36 @@ +--- +title: "Home Depot Product Search" +description: "Search Home Depot products and read prices, ratings, and product IDs." +--- + +# Home Depot Product Search + +Search The Home Depot for home improvement products. You can use the listings to track prices or add product details to a catalog. Inspect the response for your query before adding category filters. + +## Search Products + +[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. + +```python +import os +import serpapi + + +client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) + +results = client.search( + engine="home_depot", + q="table", +) + +for product in results.get("products", [])[:5]: + print(product.get("title")) + print(product.get("price"), product.get("rating")) + print(product.get("product_id")) +``` + +## Read the Results + +Read product listings from `products`. Useful fields include `title`, `product_id`, `price`, `rating`, `reviews`, `brand`, `thumbnail`, and product URLs when present. + +See [The Home Depot Search API documentation](https://serpapi.com/home-depot-search-api) for product filters and pagination. diff --git a/docs/examples/index.md b/docs/examples/index.md new file mode 100644 index 0000000..4d2ccaf --- /dev/null +++ b/docs/examples/index.md @@ -0,0 +1,205 @@ +--- +title: "Examples" +description: "Python search examples grouped by API category." +--- + +# Examples + +Each example makes a search and prints selected fields from the response. Before running one, [install the package and set your API key](../user_guide/getting-started.md#installation). + +The examples use `results.get("organic_results", [])` or a similar expression to read a list of results. The empty list `[]` is the default if that section is missing. Within a result, `item.get("title")` returns `None` if the title is missing. A slice such as `[:5]` limits the printed list to its first five items. + +Follow the API documentation linked from each example for all supported parameters. The API categories below are also listed in [SerpApi's `llms.txt`](https://serpapi.com/llms.txt). + +## Starter Examples + +### Search Engines + +- [Google Across Countries](google-across-countries.md) +- [Bing Search](bing-search.md) +- [DuckDuckGo Search](duckduckgo-search.md) +- [Baidu Search](baidu-search.md) + +### AI Answers + +- [Google AI Overview](google-ai-overview.md) + +### Local and Maps + +- [Google Maps Local Businesses](google-maps-local-business.md) +- [Google Local Services](google-local-services.md) + +### Shopping and Marketplaces + +- [Google Shopping Products](google-shopping-products.md) +- [Google Shopping Price Monitoring](google-shopping-price-monitoring.md) +- [Amazon Product Search](amazon-product-search.md) +- [Walmart Product Search](walmart-product-search.md) +- [eBay Product Listings](ebay-product-listings.md) +- [Home Depot Product Search](home-depot-product-search.md) + +### Travel and Hospitality + +- [Google Flights Travel](google-flights-travel.md) +- [Tripadvisor Travel Search](tripadvisor-travel-search.md) + +### Finance, Trends, and Jobs + +- [Google Finance Market Data](google-finance-market-data.md) +- [Google Trends Demand](google-trends-demand.md) +- [Google News Monitoring](google-news-monitoring.md) +- [Google Jobs Listings](google-jobs-listings.md) + +### Media, Apps, and Research + +- [YouTube Video Search](youtube-video-search.md) +- [Google Images Search](google-images-search.md) +- [Google Lens Image Upload](google-lens-image-upload.md) +- [Google Scholar Research](google-scholar-research.md) +- [Google Play Store Apps](google-play-store-apps.md) + +## Supported API Map + +**Search Engines** + +- [Google Search API](https://serpapi.com/search-api) +- [Google Light Search API](https://serpapi.com/google-light-api) +- [Bing Search API](https://serpapi.com/bing-search-api) +- [DuckDuckGo Search API](https://serpapi.com/duckduckgo-search-api) +- [DuckDuckGo Light API](https://serpapi.com/duckduckgo-light-api) +- [Baidu Search API](https://serpapi.com/baidu-search-api) +- [Yahoo! Search API](https://serpapi.com/yahoo-search-api) +- [Yandex Search API](https://serpapi.com/yandex-search-api) +- [Naver Search API](https://serpapi.com/naver-search-api) + +**AI Answers** + +- [Google AI Mode API](https://serpapi.com/google-ai-mode-api) +- [Google AI Overview API](https://serpapi.com/google-ai-overview-api) +- [Bing Copilot API](https://serpapi.com/bing-copilot-api) +- [Brave AI Mode API](https://serpapi.com/brave-ai-mode-api) +- [Naver AI Overview API](https://serpapi.com/naver-ai-overview-api) + +**Local and Maps** + +- [Google Local API](https://serpapi.com/google-local-api) +- [Google Local Services API](https://serpapi.com/google-local-services-api) +- [Google Maps API](https://serpapi.com/google-maps-api) +- [Google Maps Photos API](https://serpapi.com/google-maps-photos-api) +- [Google Maps Autocomplete API](https://serpapi.com/google-maps-autocomplete-api) +- [Google Maps Directions API](https://serpapi.com/google-maps-directions-api) +- [Google Maps Posts API](https://serpapi.com/google-maps-posts-api) +- [Google Maps Reviews API](https://serpapi.com/google-maps-reviews-api) +- [Google Maps Contributor Reviews API](https://serpapi.com/google-maps-contributor-reviews-api) +- [Apple Maps API](https://serpapi.com/apple-maps-api) +- [Apple Maps Places API](https://serpapi.com/apple-maps-places-api) +- [Bing Maps API](https://serpapi.com/bing-maps-api) +- [DuckDuckGo Maps API](https://serpapi.com/duckduckgo-maps-api) +- [Yelp Search API](https://serpapi.com/yelp-search-api) +- [Yelp Place API](https://serpapi.com/yelp-place) +- [Yelp Reviews API](https://serpapi.com/yelp-reviews-api) + +**Shopping and Marketplaces** + +- [Google Shopping API](https://serpapi.com/google-shopping-api) +- [Google Shopping Light API](https://serpapi.com/google-shopping-light-api) +- [Google Immersive Product API](https://serpapi.com/google-immersive-product-api) +- [Amazon Search API](https://serpapi.com/amazon-search-api) +- [Amazon Product API](https://serpapi.com/amazon-product-api) +- [Walmart Search API](https://serpapi.com/walmart-search-api) +- [Walmart Product API](https://serpapi.com/walmart-product-api) +- [Walmart Reviews API](https://serpapi.com/walmart-product-reviews-api) +- [eBay Search API](https://serpapi.com/ebay-search-api) +- [eBay Product API](https://serpapi.com/ebay-product-api) +- [The Home Depot Search API](https://serpapi.com/home-depot-search-api) +- [The Home Depot Product API](https://serpapi.com/home-depot-product) +- [The Home Depot Reviews API](https://serpapi.com/home-depot-product-reviews) +- [Bing Shopping API](https://serpapi.com/bing-shopping-api) +- [Bing Product API](https://serpapi.com/bing-product-api) +- [Yahoo! Shopping API](https://serpapi.com/yahoo-shopping-search-api) + +**Travel and Hospitality** + +- [Google Flights API](https://serpapi.com/google-flights-api) +- [Google Flights Autocomplete API](https://serpapi.com/google-flights-autocomplete-api) +- [Google Flights Deals API](https://serpapi.com/google-flights-deals-api) +- [Google Hotels API](https://serpapi.com/google-hotels-api) +- [Google Hotels Autocomplete API](https://serpapi.com/google-hotels-autocomplete-api) +- [Google Hotels Photos API](https://serpapi.com/google-hotels-photos-api) +- [Google Hotels Reviews API](https://serpapi.com/google-hotels-reviews-api) +- [Google Travel Explore API](https://serpapi.com/google-travel-explore-api) +- [Tripadvisor Search API](https://serpapi.com/tripadvisor-search-api) +- [Tripadvisor Place API](https://serpapi.com/tripadvisor-place-api) +- [Tripadvisor Reviews API](https://serpapi.com/tripadvisor-reviews-api) +- [OpenTable Reviews API](https://serpapi.com/open-table-reviews-api) + +**News, Trends, and Demand** + +- [Google News API](https://serpapi.com/google-news-api) +- [Google News Light API](https://serpapi.com/google-news-light-api) +- [Bing News API](https://serpapi.com/bing-news-api) +- [DuckDuckGo News API](https://serpapi.com/duckduckgo-news-api) +- [Baidu News API](https://serpapi.com/baidu-news-api) +- [Google Trends API](https://serpapi.com/google-trends-api) +- [Google Trends Autocomplete API](https://serpapi.com/google-trends-autocomplete) +- [Google Trends Trending Now API](https://serpapi.com/google-trends-trending-now) +- [Google Sports API](https://serpapi.com/google-sports-api) +- [Google Forums API](https://serpapi.com/google-forums-api) + +**Media, Apps, and Social** + +- [Google Images API](https://serpapi.com/google-images-api) +- [Google Images Light API](https://serpapi.com/google-images-light-api) +- [Google Lens API](https://serpapi.com/google-lens-api) +- [Google Reverse Image API](https://serpapi.com/google-reverse-image) +- [Google Videos API](https://serpapi.com/google-videos-api) +- [Google Videos Light API](https://serpapi.com/google-videos-light-api) +- [Google Short Videos API](https://serpapi.com/google-short-videos-api) +- [Bing Images API](https://serpapi.com/bing-images-api) +- [Bing Reverse Image API](https://serpapi.com/bing-reverse-image-api) +- [Bing Videos API](https://serpapi.com/bing-videos-api) +- [Yahoo! Images API](https://serpapi.com/yahoo-images-api) +- [Yahoo! Videos API](https://serpapi.com/yahoo-videos-api) +- [Yandex Images API](https://serpapi.com/yandex-images-api) +- [Yandex Videos API](https://serpapi.com/yandex-videos-api) +- [YouTube Search API](https://serpapi.com/youtube-search-api) +- [YouTube Video API](https://serpapi.com/youtube-video-api) +- [YouTube Video Transcript API](https://serpapi.com/youtube-video-transcript) +- [Apple App Store API](https://serpapi.com/apple-app-store) +- [Apple App Store Reviews API](https://serpapi.com/apple-reviews) +- [Apple App Store Product API](https://serpapi.com/apple-product) +- [Google Play Store API](https://serpapi.com/google-play-api) +- [Google Play Games API](https://serpapi.com/google-play-games) +- [Google Play Movies API](https://serpapi.com/google-play-movies) +- [Google Play Books API](https://serpapi.com/google-play-books) +- [Google Play Product API](https://serpapi.com/google-play-product-api) +- [Facebook Profile API](https://serpapi.com/facebook-profile-api) +- [Instagram Profile API](https://serpapi.com/instagram-profile-api) + +**Research, Knowledge, Ads, Finance, and Jobs** + +- [Google Scholar API](https://serpapi.com/google-scholar-api) +- [Google Scholar Author API](https://serpapi.com/google-scholar-author-api) +- [Google Scholar Case Law API](https://serpapi.com/google-scholar-case-law-api) +- [Google Patents API](https://serpapi.com/google-patents-api) +- [Google Patents Details API](https://serpapi.com/google-patents-details-api) +- [Google Related Questions API](https://serpapi.com/google-related-questions-api) +- [Google Autocomplete API](https://serpapi.com/google-autocomplete-api) +- [Google Ads API](https://serpapi.com/google-ads-api) +- [Google Ads Transparency API](https://serpapi.com/google-ads-transparency-center-api) +- [Google Finance API](https://serpapi.com/google-finance-api) +- [Google Finance Markets API](https://serpapi.com/google-finance-markets) +- [Google Jobs API](https://serpapi.com/google-jobs-api) + +```{toctree} +:hidden: +:maxdepth: 2 + +categories/search-engines +categories/ai-answers +categories/local-and-maps +categories/shopping-and-marketplaces +categories/travel-and-hospitality +categories/finance-trends-and-jobs +categories/media-apps-and-research +``` diff --git a/docs/examples/tripadvisor-travel-search.md b/docs/examples/tripadvisor-travel-search.md new file mode 100644 index 0000000..8bccc4a --- /dev/null +++ b/docs/examples/tripadvisor-travel-search.md @@ -0,0 +1,37 @@ +--- +title: "Tripadvisor Travel Search" +description: "Search Tripadvisor for destinations, hotels, restaurants, and attractions." +--- + +# Tripadvisor Travel Search + +Search Tripadvisor for destinations, hotels, restaurants, attractions, and forum posts. Set `ssrc` to limit the search to one type of result, such as hotels. + +## Search Places + +[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. + +```python +import os +import serpapi + + +client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) + +results = client.search( + engine="tripadvisor", + q="Rome", + ssrc="h", +) + +for place in results.get("places", [])[:5]: + print(place.get("title")) + print(place.get("place_type"), place.get("location")) + print(place.get("link")) +``` + +## Read the Results + +Read the returned places from `places`. Useful fields include `title`, `place_type`, `place_id`, `location`, `description`, `thumbnail`, `link`, and `serpapi_link`. + +Choose the result type with `ssrc`: `h` for hotels, `r` for restaurants, `A` for things to do, `g` for destinations, and `a` for all results. See the [Tripadvisor Search API documentation](https://serpapi.com/tripadvisor-search-api) for the current filter and pagination options. diff --git a/docs/examples/walmart-product-search.md b/docs/examples/walmart-product-search.md new file mode 100644 index 0000000..fdfdb48 --- /dev/null +++ b/docs/examples/walmart-product-search.md @@ -0,0 +1,38 @@ +--- +title: "Walmart Product Search" +description: "Search Walmart products and read prices, reviews, shipping details, and product IDs." +--- + +# Walmart Product Search + +Search Walmart for products, prices, and availability. You can use the results to track prices or add details to a product catalog. Set `query` to your search text. This engine does not use `q`. + +## Search Products + +[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. + +```python +import os +import serpapi + + +client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) + +results = client.search( + engine="walmart", + query="coffee maker", + walmart_domain="walmart.com", +) + +for product in results.get("organic_results", [])[:5]: + offer = product.get("primary_offer", {}) + print(product.get("title")) + print(offer.get("offer_price"), product.get("rating"), product.get("reviews")) + print(product.get("us_item_id"), product.get("product_page_url")) +``` + +## Read the Results + +Read product listings from `organic_results`. Useful fields include `title`, `us_item_id`, `product_id`, `rating`, `reviews`, `seller_name`, `primary_offer.offer_price`, `price_per_unit`, `out_of_stock`, `product_page_url`, and `serpapi_product_page_url`. + +Use `sort`, `min_price`, `max_price`, `store_id`, and `facet` to sort results, set price limits, or filter by store and product features. See the [Walmart Search API documentation](https://serpapi.com/walmart-search-api) for supported filters and pagination behavior. diff --git a/docs/examples/youtube-video-search.md b/docs/examples/youtube-video-search.md new file mode 100644 index 0000000..ce8d5e7 --- /dev/null +++ b/docs/examples/youtube-video-search.md @@ -0,0 +1,36 @@ +--- +title: "YouTube Video Search" +description: "Search YouTube and read video titles, channels, and links." +--- + +# YouTube Video Search + +Search YouTube for videos about a topic and read their channel details. Set `search_query` to your search text. This engine does not use `q`. + +## Search Videos + +[Install the package and set your API key](../user_guide/getting-started.md#installation) before running this example. + +```python +import os +import serpapi + + +client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) + +results = client.search( + engine="youtube", + search_query="coffee brewing guide", +) + +for video in results.get("video_results", [])[:5]: + print(video.get("title")) + print(video.get("channel", {}).get("name")) + print(video.get("link")) +``` + +## Read the Results + +Read videos from `video_results`. Useful fields include `title`, `link`, `channel`, `views`, `published_date`, `length`, and `thumbnail`. Some queries return additional sections such as channels, playlists, shorts, or related searches. + +See the [YouTube Search API documentation](https://serpapi.com/youtube-search-api) for YouTube-specific parameters. The [SerpApi Playground](https://serpapi.com/playground) lets you inspect the response fields for your query. diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..f5c4537 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,163 @@ +--- +title: "SerpApi Python Library & Package" +description: "Search Google and other engines from Python with the official SerpApi client." +--- + +# SerpApi Python Library & Package + +`serpapi` is the official Python client for [SerpApi](https://serpapi.com). Use it to search the web and read the results in your Python programs. + +SerpApi supports Google, Google Maps, Google Shopping, Bing, DuckDuckGo, Baidu, Yandex, Yahoo, eBay, YouTube, App Stores, Walmart, Home Depot, Naver, and many more engines. + +You can retrieve web search results, local business listings, shopping results, flight schedules, stock market data, job listings, trends, news headlines, AI Overview answers, and video search results. + +## Install + +Run one of these commands in your terminal. Use `pip` to install into your Python environment, or `uv add` if you manage your project with uv. + +::::{tab-set} + +:::{tab-item} pip + +```bash +pip install serpapi +``` + +::: + +:::{tab-item} uv + +```bash +uv add serpapi +``` + +::: + +:::: + +The package requires Python 3.6 or newer. + +## First Request + +Sign up at [SerpApi](https://serpapi.com/users/sign_up) and copy your API key from the [dashboard](https://serpapi.com/manage-api-key). Replace `secret_api_key` below with your key, then run the command in your terminal. On Windows, use PowerShell. + +::::{tab-set} + +:::{tab-item} macOS / Linux + +```bash +export SERPAPI_KEY="secret_api_key" +``` + +::: + +:::{tab-item} Windows + +```powershell +$env:SERPAPI_KEY = "secret_api_key" +``` + +::: + +:::: + +Run this code in a Python script or interpreter started from the same terminal so it can read `SERPAPI_KEY`: + +```python +import os +import serpapi + +client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"]) +results = client.search( + engine="google", + q="coffee", + location="Austin, Texas", + hl="en", + gl="us", +) + +print(results["organic_results"][0]["link"]) +``` + +This prints the link from the first organic result, which is an unpaid search listing. `results["organic_results"]` is a list, and `[0]` selects its first item. See [Getting Started](user_guide/getting-started.md) for setup instructions and an explanation of each parameter. + +The `results` variable contains a `SerpResults` object. You can read its fields like a Python dictionary, convert it to a plain dictionary, or use its methods to fetch more pages. You can also retrieve saved searches with `client.search_archive()`. + +Use the same search parameter names as the [SerpApi API documentation](https://serpapi.com/search-api). The [SerpApi Playground](https://serpapi.com/playground) lets you try a search in your browser and copy its parameters into Python. + +## Documentation Map + +- [Getting Started](user_guide/getting-started.md) covers installation and your first search. [Client Usage](user_guide/client-usage.md) explains how to make requests and read responses. +- [AI Agents](ai-agents.md) covers search tools, Markdown results, and API references for agents. +- [Output Formats](user_guide/output-formats.md) explains when to use JSON, Markdown, or HTML. +- [Migration Guide](user_guide/migrating-from-google-search-results.md) shows how to replace `google-search-results` with this package. +- [Parameters and Engines](user_guide/parameters-and-engines.md) explains search parameters and how to test them in the [SerpApi Playground](https://serpapi.com/playground). +- For scripts that collect results, see [pagination](user_guide/pagination.md), [timeouts and errors](user_guide/errors-and-timeouts.md), [request options](user_guide/request-options.md), and [account and locations](user_guide/account-and-locations.md). +- For searches you retrieve later, selected response fields, and data retention settings, see [Async Search Archive](user_guide/async-search-archive.md), [JSON Restrictor](user_guide/json-restrictor.md), and [Zero Trace](user_guide/zero-trace.md). +- [Examples](examples/index.md) includes searches for web pages, AI answers, local businesses, products, travel, finance, trends, jobs, media, apps, and research. + +```{toctree} +:hidden: +:maxdepth: 2 +:caption: Basics + +user_guide/getting-started +user_guide/client-usage +``` + +```{toctree} +:hidden: +:maxdepth: 2 +:caption: User Guide + +User Guide Overview +user_guide/parameters-and-engines +user_guide/output-formats +user_guide/pagination +user_guide/errors-and-timeouts +user_guide/account-and-locations +``` + +```{toctree} +:hidden: +:maxdepth: 2 +:caption: Migration + +user_guide/migrating-from-google-search-results +``` + +```{toctree} +:hidden: +:maxdepth: 2 +:caption: Advanced Usage + +user_guide/async-search-archive +user_guide/threading +user_guide/multiprocessing +user_guide/zero-trace +user_guide/json-restrictor +user_guide/request-options +``` + +```{toctree} +:hidden: +:maxdepth: 2 +:caption: AI Agents + +ai-agents +``` + +```{toctree} +:hidden: +:maxdepth: 3 + +examples/index +``` + +```{toctree} +:hidden: +:maxdepth: 2 +:caption: Reference + +reference +``` diff --git a/docs/index.rst b/docs/index.rst deleted file mode 100644 index de3b83b..0000000 --- a/docs/index.rst +++ /dev/null @@ -1,193 +0,0 @@ -.. serpapi-python documentation master file, created by - sphinx-quickstart on Sun Apr 3 21:09:40 2022. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -**serpapi-python** -================== - -an official Python client library for `SerpApi `_. - --------------- - -Installation ------------- - -To install ``serpapi-python``, simply use `pip`:: - - $ pip install serpapi - - -Please note that Python 3.6+ is required. - - -Usage ------ - -Usage of this module is fairly straight-forward. In general, this module attempts to be as close to the actual API as possible, while still being Pythonic. - -For example, the API endpoint ``https://serpapi.com/search.json`` is represented by the method ``serpapi.search()``. - -.. code-block:: python - - >>> import serpapi - >>> s = serpapi.search(q="Coffee", engine="google", location="Austin, Texas", hl="en", gl="us") - >>> s["organic_results"][0]["link"] - 'https://en.wikipedia.org/wiki/Coffee' - -Any parameters that you pass to ``search()`` will be passed to the API. This includes the ``api_key`` parameter, which is required for all requests. - -.. _using-api-client-directly: - -Using the API Client directly -^^^^^^^^^ - -To make this less repetitive, and gain the benefit of connection pooling, let's start using the API Client directly:: - - >>> client = serpapi.Client(api_key="secret_api_key") - >>> s = client.search(q="Coffee", engine="google", location="Austin, Texas", hl="en", gl="us") - -The ``api_key`` parameter is now automatically passed to all requests made by the client. - - -Concise Tutorial ----------------- - -Let's start by searching for ``Coffee`` on Google:: - - >>> import serpapi - >>> s = serpapi.search(q="Coffee", engine="google", location="Austin, Texas", hl="en", gl="us") - -The ``s`` variable now contains a :class:`SerpResults ` object, which acts just like a standard dictionary, with some convenient functions added on top. - -Let's print the first result:: - - >>> print(s["organic_results"][0]["link"]) - https://en.wikipedia.org/wiki/Coffee - -Let's print the title of the first result, but in a more Pythonic way:: - - >>> print(s["organic_results"][0].get("title")) - Coffee - Wikipedia - -The `SerpApi.com API Documentation `_ contains a list of all the possible parameters that can be passed to the API. - - -API Reference -------------- - -.. _api-reference: - -This part of the documentation covers all the interfaces of :class:`serpapi` Python module. - -.. module:: serpapi - :platform: Unix, Windows - :synopsis: SerpApi Python Library - -.. autofunction:: serpapi.search -.. autofunction:: serpapi.search_archive -.. autofunction:: serpapi.upload_image -.. autofunction:: serpapi.locations -.. autofunction:: serpapi.account - - - -Results from SerpApi.com ------------------------- - -When a successful search has been executed, the method returns -a :class:`SerpResults ` object, which acts just like a standard dictionary, -with some convenient functions added on top. - - -.. code-block:: python - - >>> s = serpapi.search(q="Coffee", engine="google", location="Austin, Texas", hl="en", gl="us") - >>> type(s) - - - >>> s["organic_results"][0]["link"] - 'https://en.wikipedia.org/wiki/Coffee' - - >>> s["search_metadata"] - {'id': '64c148d35119a60ab1e00cc9', 'status': 'Success', 'json_endpoint': 'https://serpapi.com/searches/a15e1b92727f292c/64c148d35119a60ab1e00cc9.json', 'created_at': '2023-07-26 16:24:51 UTC', 'processed_at': '2023-07-26 16:24:51 UTC', 'google_url': 'https://www.google.com/search?q=Coffee&oq=Coffee&uule=w+CAIQICIdQXVzdGluLFRYLFRleGFzLFVuaXRlZCBTdGF0ZXM&hl=en&gl=us&sourceid=chrome&ie=UTF-8', 'raw_html_file': 'https://serpapi.com/searches/a15e1b92727f292c/64c148d35119a60ab1e00cc9.html', 'total_time_taken': 1.55} - -Optionally, if you want exactly a dictionary of the entire response, you can use the ``as_dict()`` method:: - - >>> type(s.as_dict()) - - -You can get the next page of results:: - - >>> type(s.next_page()) - - -To iterate over all pages of results, it's recommended to :ref:`use the API Client directly `:: - - >>> client = serpapi.Client(api_key="secret_api_key") - >>> search = client.search(q="Coffee", engine="google", location="Austin, Texas", hl="en", gl="us") - >>> for page in search.yield_pages(): - ... print(page["search_metadata"]["page_number"]) - 1 - 2 - 3 - 4 - 5 - 6 - 7 - 8 - 9 - 10 - - -Here's documentation of the class itself and its methods: - -.. autoclass:: serpapi.SerpResults - - .. automethod:: SerpResults.next_page - .. automethod:: SerpResults.yield_pages - .. autoproperty:: SerpResults.next_page_url - - -API Client ----------- - -The primary interface to `serpapi-python` is through the :class:`serpapi.Client` class. -The primary benefit of using this class is to benefit from Requests' HTTP Connection Pooling. -This class also alleviates the need to pass an ``api_key``` along with every search made to the platform. - -.. autoclass:: serpapi.Client - - .. automethod:: Client.search - .. automethod:: Client.search_archive - .. automethod:: Client.upload_image - .. automethod:: Client.account - .. automethod:: Client.locations - - - -Exceptions ----------- - -.. autoexception:: serpapi.SerpApiError - :members: - -.. autoexception:: serpapi.SearchIDNotProvided - :members: - -.. autoexception:: serpapi.HTTPError - :members: - -.. autoexception:: serpapi.HTTPConnectionError - :members: - - - - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/docs/reference.md b/docs/reference.md new file mode 100644 index 0000000..9fbd9ce --- /dev/null +++ b/docs/reference.md @@ -0,0 +1,8 @@ +# API Reference + +Look up the arguments, return values, and exceptions for the functions and classes in `serpapi`. For examples of how to use them together, see [Client Usage](user_guide/client-usage.md). + +```{eval-rst} +.. automodule:: serpapi + :member-order: groupwise +``` diff --git a/docs/requirements.txt b/docs/requirements.txt index 987dd00..142b6ca 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,53 +1 @@ -alabaster==0.7.13 -astroid==2.15.5 -Babel==2.12.1 -bleach==6.0.0 -certifi==2023.5.7 -charset-normalizer==3.2.0 -coverage==7.2.7 -dill==0.3.6 -docutils==0.20.1 -idna==3.4 -imagesize==1.4.1 -importlib-metadata==6.7.0 -iniconfig==2.0.0 -isort==5.12.0 -jaraco.classes==3.2.3 -Jinja2==3.1.2 -keyring==24.2.0 -lazy-object-proxy==1.9.0 -markdown-it-py==3.0.0 -MarkupSafe==2.1.3 -mccabe==0.7.0 -mdurl==0.1.2 -more-itertools==9.1.0 -packaging==23.1 -pkginfo==1.9.6 -platformdirs==3.8.0 -pluggy==1.2.0 -pycodestyle==2.10.0 -Pygments==2.15.1 -pylint==2.17.4 -pytest==7.4.0 -pytest-cov==4.1.0 -readme-renderer==40.0 -requests==2.31.0 -requests-toolbelt==1.0.0 -rfc3986==2.0.0 -rich==13.4.2 -six==1.16.0 -snowballstemmer==2.2.0 -Sphinx==7.0.1 -sphinxcontrib-applehelp==1.0.4 -sphinxcontrib-devhelp==1.0.2 -sphinxcontrib-htmlhelp==2.0.1 -sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-qthelp==1.0.3 -sphinxcontrib-serializinghtml==1.1.5 -tomlkit==0.11.8 -twine==4.0.2 -urllib3==2.0.3 -webencodings==0.5.1 -wrapt==1.15.0 -zipp==3.15.0 --e . +-e .[docs] diff --git a/docs/user_guide/account-and-locations.md b/docs/user_guide/account-and-locations.md new file mode 100644 index 0000000..d48aab2 --- /dev/null +++ b/docs/user_guide/account-and-locations.md @@ -0,0 +1,47 @@ +--- +title: "Account and Locations" +description: "Check your search allowance and find locations for Google searches." +--- + +# Account and Locations + +Use `client.account()` to check your account and `client.locations()` to find locations for a Google search. + +## Account + +After [setting your API key](getting-started.md#installation), call `account()` to read your account information: + +```python +import os +import serpapi + +client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"]) + +account = client.account() +print(account) +``` + +The returned dictionary includes `total_searches_left`, `this_month_usage`, and `account_rate_limit_per_hour`. You can check these before running a batch of searches. See the [Account API documentation](https://serpapi.com/account-api) for all fields. + +## Locations + +Use `locations()` to find supported Google locations: + +```python +locations = client.locations(q="Austin", limit=5) + +for location in locations: + print(location.get("name"), location.get("canonical_name")) +``` + +Choose a `canonical_name` from the returned locations and pass it to a search. For example: + +```python +results = client.search( + engine="google", + q="coffee", + location="Austin, Texas, United States", +) +``` + +See the [SerpApi Locations API documentation](https://serpapi.com/locations-api) for supported parameters and response fields. diff --git a/docs/user_guide/async-search-archive.md b/docs/user_guide/async-search-archive.md new file mode 100644 index 0000000..f669653 --- /dev/null +++ b/docs/user_guide/async-search-archive.md @@ -0,0 +1,59 @@ +--- +title: "Async Search Archive" +description: "Submit searches now and retrieve their results later with the Search Archive API." +--- + +# Async Search Archive + +An asynchronous search lets you submit a request now and collect the results later. Set the API's `async` parameter to `true` to receive a search ID, then use that ID with the Search Archive API. Each Python client call still waits for an HTTP response. These methods do not use Python's `asyncio` or `await`. + +Create a client as shown in [Client Usage](client-usage.md#create-a-client) before running the examples. + +## Submit an Async Search + +`async` is a reserved word in Python. Put it in a dictionary and use `**` to pass that dictionary's entries as keyword arguments: + +```python +submitted = client.search( + engine="google", + q="coffee", + location="Austin, Texas", + **{"async": True}, +) + +search_id = submitted["search_metadata"]["id"] +print(search_id) +``` + +## Retrieve the Search + +```python +archived = client.search_archive(search_id=search_id) + +status = archived.get("search_metadata", {}).get("status") +print(status) +``` + +Check the status before reading the results. If the search is still processing, wait and call `search_archive()` again. This loop checks every two seconds until the search succeeds or reports an error: + +```python +import time + +while True: + archived = client.search_archive(search_id=search_id) + status = archived.get("search_metadata", {}).get("status") + + if status == "Success": + break + + if status == "Error": + raise RuntimeError(archived.get("error", "Search failed")) + + time.sleep(2) +``` + +## When to Use Async Searches + +Use asynchronous searches when one part of your program submits searches and another collects the results later. For example, a scheduled job could submit a batch of searches for another worker to process. + +Do not combine `async=true` with `no_cache=true`. See the [SerpApi Search Archive API documentation](https://serpapi.com/search-archive-api) for search statuses and how long saved searches remain available. diff --git a/docs/user_guide/client-usage.md b/docs/user_guide/client-usage.md new file mode 100644 index 0000000..8680c10 --- /dev/null +++ b/docs/user_guide/client-usage.md @@ -0,0 +1,178 @@ +--- +title: "Client Usage" +description: "Use the SerpApi client, module helpers, request options, and response helpers." +--- + +# Client Usage + +Create a `serpapi.Client` to reuse your API key, timeout, and HTTP connection settings across requests. + +## Create a Client + +First, [install the package and set `SERPAPI_KEY`](getting-started.md#installation). The examples on this page use the following client: + +```python +import os +import serpapi + +client = serpapi.Client( + api_key=os.environ["SERPAPI_KEY"], + timeout=20, +) +``` + +`timeout=20` sets the default connection and read timeout in seconds for this client's requests. It applies to searches, archive lookups, account and location requests, and image uploads. See [Errors and Timeouts](errors-and-timeouts.md#connection-errors-and-timeouts) for how the timeout works. + +## Search + +Pass search parameters by name, using Python keyword arguments: + +```python +results = client.search(engine="google", q="coffee") +``` + +You can also pass a dictionary when parameters are already stored in one: + +```python +params = { + "engine": "google", + "q": "coffee", +} +results = client.search(params) +``` + +You can combine a dictionary with keyword arguments. The client adds the keyword arguments to the dictionary and replaces existing values with the same name: + +```python +params = {"engine": "google", "q": "coffee"} +results = client.search(params, location="Austin, Texas") +``` + +## Module Helpers + +You can call functions such as `serpapi.search()` directly without creating a client: + +```python +import serpapi + +results = serpapi.search( + api_key="secret_api_key", + engine="google", + q="coffee", +) +``` + +These functions use a shared client. Create your own `serpapi.Client` when you need separate settings for an application, test, or worker that runs searches. + +## Search Archive + +Use `search_archive()` to retrieve a saved search. Get its ID from the original response: + +```python +search_id = results["search_metadata"]["id"] +archived = client.search_archive(search_id=search_id) +``` + +If `search_id` is missing, the client raises `serpapi.SearchIDNotProvided`. + +## Account and Locations + +```python +account = client.account() +locations = client.locations(q="Austin", limit=3) +``` + +`account()` returns account information for your API key. `locations()` returns supported Google locations that match the query. See [Account and Locations](account-and-locations.md) for examples and how to use a location in a search. + +## Request Options + +`search()`, `search_archive()`, `account()`, and `locations()` pass these keyword arguments to the underlying `requests` call: + +| Option | Use | +| --- | --- | +| `timeout` | Set a different timeout, in seconds, for one request. | +| `proxies` | Send the request through a proxy. | +| `verify` | Check the server's TLS certificate, or use a custom certificate authority bundle. | +| `stream` | Set the `requests` streaming option. The client still reads the response before returning results. | +| `cert` | Authenticate the request with a client certificate. | + +For example, set a shorter timeout for this search: + +```python +results = client.search( + engine="google", + q="coffee", + timeout=10, +) +``` + +See [Request Options](request-options.md) for proxy, TLS, certificate, and per-request timeout examples. + +## Response Objects + +For JSON searches, the client returns a `serpapi.SerpResults` object. Read its fields like a dictionary: + +```python +first = results["organic_results"][0] +print(first.get("title")) +print(first.get("link")) +``` + +Use `as_dict()` when another library needs a plain dictionary: + +```python +payload = results.as_dict() +``` + +Use `output="html"` to receive the original search results page as an HTML string: + +```python +html = client.search(engine="google", q="coffee", output="html") +print(html[:500]) +``` + +Use `output="md"` to receive a Markdown string, for example to pass search results to an AI agent: + +```python +markdown = client.search(engine="google", q="coffee", output="md") +print(markdown[:500]) +``` + +See [Output Formats](output-formats.md) for details on JSON, Markdown, and HTML responses. + +## Pagination Helpers + +For one additional page, call `next_page()`: + +```python +next_results = results.next_page() +``` + +Use `yield_pages()` in a loop to read the current page and fetch more pages. This example stops after ten pages, or earlier if there is no next page: + +```python +for page_number, page in enumerate(results.yield_pages(max_pages=10), start=1): + current = page.get("serpapi_pagination", {}).get("current", page_number) + print(current) +``` + +See [Pagination](pagination.md) for more examples and how to use Google's `start` parameter. + +## Error Handling + +Use `try` and `except` to handle a timeout, a connection failure, or an HTTP error: + +```python +import serpapi + +try: + results = client.search(engine="google", q="coffee") +except serpapi.TimeoutError: + print("The request timed out.") +except serpapi.HTTPConnectionError: + print("Could not connect to SerpApi.") +except serpapi.HTTPError as exc: + print(exc.status_code, exc.error) +``` + +See [Errors and Timeouts](errors-and-timeouts.md) for error details and timeout settings. diff --git a/docs/user_guide/errors-and-timeouts.md b/docs/user_guide/errors-and-timeouts.md new file mode 100644 index 0000000..58a2814 --- /dev/null +++ b/docs/user_guide/errors-and-timeouts.md @@ -0,0 +1,81 @@ +--- +title: "Errors and Timeouts" +description: "Handle SerpApi HTTP errors, connection errors, timeouts, and API-level errors." +--- + +# Errors and Timeouts + +The client raises exceptions when a request times out, cannot connect, or receives an HTTP error. Use `try` and `except` to decide what your script should do when a request fails. + +The examples use a client created as shown in [Client Usage](client-usage.md#create-a-client). + +## HTTP Errors + +HTTP responses with a 4xx or 5xx status code raise `serpapi.HTTPError`: + +```python +import serpapi + +try: + results = client.search(engine="google", q="coffee") +except serpapi.HTTPError as exc: + print("Status:", exc.status_code) + print("Error:", exc.error) +``` + +`exc.status_code` contains the HTTP status code, such as `401` for an invalid API key. `exc.error` contains the error message from SerpApi's JSON response, or `None` if the response has no JSON error message. + +See [SerpApi API status and error codes](https://serpapi.com/api-status-and-error-codes) for the meaning of each status and how to resolve it. + +## Connection Errors and Timeouts + +Catch `TimeoutError` when a request takes too long to connect or receive data. Catch `HTTPConnectionError` when the client cannot establish a connection: + +```python +try: + results = client.search(engine="google", q="coffee", timeout=10) +except serpapi.TimeoutError: + print("The request timed out.") +except serpapi.HTTPConnectionError: + print("Could not connect to SerpApi.") +``` + +Timeouts are measured in seconds. Set a default for all requests made by a client: + +```python +client = serpapi.Client(api_key="secret_api_key", timeout=20) +``` + +Override it for one request: + +```python +results = client.search( + engine="google", + q="coffee", + timeout=5, +) +``` + +The timeout applies to connecting and waiting for data. A request can take longer than the timeout overall if data continues to arrive. Without a timeout setting, the client can wait indefinitely. See the [Requests timeout documentation](https://requests.readthedocs.io/en/latest/user/quickstart/#timeouts). + +## Missing Search IDs + +`search_archive()` requires the ID of a previous search. If you leave out `search_id`, it raises `SearchIDNotProvided`: + +```python +try: + archived = client.search_archive() +except serpapi.SearchIDNotProvided: + print("Provide search_id from search_metadata.id.") +``` + +## API-Level Errors in JSON + +A response can contain an `error` message even when the HTTP request succeeds. Check for this field before reading the results: + +```python +results = client.search(engine="google", q="coffee") + +if "error" in results: + raise RuntimeError(results["error"]) +``` diff --git a/docs/user_guide/getting-started.md b/docs/user_guide/getting-started.md new file mode 100644 index 0000000..99490b1 --- /dev/null +++ b/docs/user_guide/getting-started.md @@ -0,0 +1,136 @@ +--- +title: "Getting Started" +description: "Install serpapi, create a client, and run your first search." +--- + +# Getting Started + +The `serpapi` package lets you request search results from [SerpApi](https://serpapi.com) in Python. You need Python installed and a SerpApi account with an API key. + +## Installation + +Run one of these commands in your terminal. Use `pip` for an existing Python environment, `uv add` for a uv project, or `uv pip install` for a virtual environment managed with uv. + +::::{tab-set} + +:::{tab-item} pip + +```bash +pip install serpapi +``` + +::: + +:::{tab-item} uv project + +```bash +uv add serpapi +``` + +::: + +:::{tab-item} uv environment + +```bash +uv pip install serpapi +``` + +::: + +:::: + +The package requires Python 3.6 or newer. + +Create or sign in to your SerpApi account and copy your API key from the [dashboard](https://serpapi.com/manage-api-key). An API key identifies your account when you make a request. Replace `secret_api_key` below with your key and run the command in your terminal. On Windows, use PowerShell. + +::::{tab-set} + +:::{tab-item} macOS / Linux + +```bash +export SERPAPI_KEY="secret_api_key" +``` + +::: + +:::{tab-item} Windows + +```powershell +$env:SERPAPI_KEY = "secret_api_key" +``` + +::: + +:::: + +## First Search + +Save this code in a file named `search_example.py`. Run it with `python search_example.py` from the same terminal where you set the key. Use `python3` if that is the command for Python on your computer. If you installed the package with `uv add`, run `uv run python search_example.py` from your project directory. + +```python +import os +import serpapi + +client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) + +results = client.search( + engine="google", + q="coffee shops", + location="Austin, Texas", + hl="en", + gl="us", +) + +first_result = results["organic_results"][0] +print(first_result["title"]) +print(first_result["link"]) +``` + +`os.environ["SERPAPI_KEY"]` reads the key you set in the terminal. If Python raises `KeyError: 'SERPAPI_KEY'`, set the variable in the terminal where you run the script, or in your editor's run configuration. + +`client.search()` sends the search to SerpApi. By default, the response uses JSON, a format for named fields and lists. The client converts it into a `SerpResults` object, which you can read like a Python dictionary. It also has methods such as `as_dict()`, `next_page()`, and `yield_pages()`. + +`results["organic_results"]` is a list of unpaid search results. Python counts list positions from zero, so `[0]` selects the first result. The example prints its title and link. For searches that may return no results, use the loop shown in [Output Formats](output-formats.md#json). + +## What the Parameters Mean + +The example above sends a Google Search request: + +| Parameter | Purpose | +| --- | --- | +| `engine` | Selects the SerpApi engine. `google` is Google Search. | +| `q` | Text to search for, such as `coffee shops`. | +| `location` | Location to search from, such as `Austin, Texas`. | +| `hl` | Language for the Google interface. `en` is English. | +| `gl` | Country for Google results. `us` is the United States. | + +Each engine has its own search parameters, listed in the [SerpApi API documentation](https://serpapi.com/search-api). Use the [SerpApi Playground](https://serpapi.com/playground) to try a search in your browser and copy the parameters into Python. + +## When to Use the Client + +Create a `serpapi.Client` when you need to make several requests, fetch more pages, or run searches at the same time. You can reuse its API key, timeout, and HTTP session, which manages connections to SerpApi. + +For a single search, you can also call `serpapi.search()` directly: + +```python +import serpapi + +results = serpapi.search( + api_key="secret_api_key", + engine="google", + q="coffee", +) +``` + +To reuse the same settings for later searches, create a client: + +```python +client = serpapi.Client(api_key="secret_api_key", timeout=20) +results = client.search(engine="google", q="coffee") +``` + +## Next Steps + +- Read [Client Usage](client-usage.md) for response helpers, request options, and archive helpers. +- Read [Pagination](pagination.md) when collecting more than one page of results. +- Try [Google Across Countries](../examples/google-across-countries.md) to compare results from different countries. diff --git a/docs/user_guide/index.md b/docs/user_guide/index.md new file mode 100644 index 0000000..9e1fd5b --- /dev/null +++ b/docs/user_guide/index.md @@ -0,0 +1,29 @@ +# User Guide + +Start with [Getting Started](getting-started.md) to install the package and run a search. The other guides explain how to read results, fetch more pages, and handle errors. + +## Basics + +- [Getting Started](getting-started.md) +- [Client Usage](client-usage.md) + +## User Guide + +- [Parameters and Engines](parameters-and-engines.md) +- [Output Formats](output-formats.md) +- [Pagination](pagination.md) +- [Errors and Timeouts](errors-and-timeouts.md) +- [Account and Locations](account-and-locations.md) + +## Migration + +- [Migrating from google-search-results](migrating-from-google-search-results.md) + +## Advanced Usage + +- [Async Search Archive](async-search-archive.md) +- [Threading](threading.md) +- [Multiprocessing](multiprocessing.md) +- [Zero Trace and Data Retention](zero-trace.md) +- [JSON Restrictor](json-restrictor.md) +- [Request Options](request-options.md) diff --git a/docs/user_guide/json-restrictor.md b/docs/user_guide/json-restrictor.md new file mode 100644 index 0000000..99506df --- /dev/null +++ b/docs/user_guide/json-restrictor.md @@ -0,0 +1,137 @@ +--- +title: "JSON Restrictor" +description: "Request selected JSON fields with json_restrictor." +--- + +# JSON Restrictor + +Use `json_restrictor` to ask SerpApi to return selected fields. For example, you can request titles and links while leaving out images and other sections you do not need. SerpApi filters the response before sending it to your program. + +The parameter works across SerpApi engines. Its value is a selector string that names the fields to keep. See the [SerpApi JSON Restrictor documentation](https://serpapi.com/json-restrictor) for all selector operators. + +Create a client as shown in [Client Usage](client-usage.md#create-a-client) before running the examples. + +## Select a Top-Level Section + +Return only `organic_results`: + +```python +results = client.search( + engine="google", + q="coffee", + location="Austin, Texas", + json_restrictor="organic_results", +) + +for result in results.get("organic_results", []): + print(result.get("title")) +``` + +## Select Fields From Each Result + +Use `[]` to select every item in a JSON array, which becomes a list in Python. List field names inside `.{...}` to keep several fields from each item: + +```python +results = client.search( + engine="google", + q="coffee", + location="Austin, Texas", + json_restrictor="organic_results[].{title, link, snippet}", +) + +for result in results.get("organic_results", []): + print(result["title"], result["link"]) +``` + +The response keeps the `organic_results` list. Each result in that list contains only the selected fields that are present. + +## Select One Item or a Slice + +Use `[0]` to select the first item in a list. As in Python, indexes start at zero: + +```python +results = client.search( + engine="google", + q="coffee", + json_restrictor="organic_results[0]", +) + +first_result = results["organic_results"][0] +print(first_result["title"]) +``` + +Use a slice to select a range of items. The start index is included and the end index is excluded: + +```python +results = client.search( + engine="google", + q="coffee", + json_restrictor="organic_results[0:3]", +) +``` + +`organic_results[0:3]` keeps indexes `0`, `1`, and `2`, so it returns at most three items. + +## Combine Multiple Selectors + +Separate selectors with commas to keep more than one part of the response: + +```python +results = client.search( + engine="google", + q="coffee", + location="Austin, Texas", + json_restrictor="local_map, organic_results[0]", +) + +print(results.get("local_map", {}).get("gps_coordinates")) +print(results["organic_results"][0]["title"]) +``` + +## Nested Fields + +Use dots to select fields inside other fields. This selector keeps each result's title and the links in its `sitelinks.inline` list: + +```python +results = client.search( + engine="google", + q="coffee", + json_restrictor="organic_results[].{title, sitelinks.inline[].link}", +) +``` + +The selected links remain inside `sitelinks.inline`, so your code can read them at the same location as in the full response. + +## Common Selector Patterns + +| Need | `json_restrictor` | +| --- | --- | +| One top-level section | `organic_results` | +| First organic result | `organic_results[0]` | +| First three organic results | `organic_results[0:3]` | +| One field from every result | `organic_results[].title` | +| Multiple fields from every result | `organic_results[].{title, snippet}` | +| Nested fields | `organic_results[].{title, sitelinks.inline[].link}` | +| Multiple response sections | `local_map, organic_results[0]` | + +## Practical Guidance + +- First make a search without `json_restrictor` and inspect the response. Then add selectors for the fields you need. +- Check selectors against a response from the engine you are using. Result sections and field names vary between engines. +- If you plan to call `next_page()` or `yield_pages()`, include `serpapi_pagination` in the selector so the client can find the next-page URL. +- If you need a search ID, status, or archive metadata, include `search_metadata`. + +To fetch more pages, keep `serpapi_pagination` alongside the result fields: + +```python +results = client.search( + engine="google", + q="coffee", + location="Austin, Texas", + json_restrictor="organic_results[].{title, link}, serpapi_pagination", +) + +for page in results.yield_pages(max_pages=3): + for result in page.get("organic_results", []): + print(result["title"], result["link"]) +``` diff --git a/docs/user_guide/migrating-from-google-search-results.md b/docs/user_guide/migrating-from-google-search-results.md new file mode 100644 index 0000000..e074a1b --- /dev/null +++ b/docs/user_guide/migrating-from-google-search-results.md @@ -0,0 +1,114 @@ +--- +title: "Migrating from google-search-results" +description: "Move from the deprecated google-search-results SDK to the recommended serpapi package." +--- + +# Migrating from google-search-results + +Both `google-search-results` and `serpapi` use the name `serpapi` in Python imports. Check which package your project installs before updating the code. + +## Recommended Package + +Use [`serpapi` on PyPI](https://pypi.org/project/serpapi/) for new projects and when updating an existing integration. + +Install it with: + +```bash +pip install serpapi +``` + +After [setting your API key](getting-started.md#installation), create a `serpapi.Client`: + +```python +import os +import serpapi + +YOUR_API_KEY = os.environ["SERPAPI_KEY"] + +client = serpapi.Client(api_key=YOUR_API_KEY) +results = client.search(engine="google", q="coffee") + +print(results) +``` + +This is the package used throughout the current documentation. + +## Deprecated Package + +[`google-search-results` on PyPI](https://pypi.org/project/google-search-results/) is the older Python package. It is deprecated for new integrations. + +It was installed with: + +```bash +pip install google-search-results +``` + +Older code often looks like this: + + +```python +from serpapi import GoogleSearch + +search = GoogleSearch({ + "q": "coffee", + "location": "Austin,Texas", + "api_key": "", +}) +result = search.get_dict() +``` + +Do not add this package to new projects. + +## Update Your Dependencies + +Remove `google-search-results` from your dependency files, such as `requirements.txt`, `pyproject.toml`, `setup.py`, `Pipfile`, or Poetry dependency configuration. + +Then add `serpapi` instead: + +```bash +pip install serpapi +``` + +Install only one of these packages in a Python environment. Both provide a `serpapi` module, so installing them together can overwrite files and cause import errors. + +## Update Your Code + +Replace `GoogleSearch(...).get_dict()` with `serpapi.Client(...).search(...)`. + +Old: + + +```python +from serpapi import GoogleSearch + +search = GoogleSearch({ + "q": "coffee", + "location": "Austin,Texas", + "api_key": "", +}) +result = search.get_dict() +``` + +New: + +```python +import os +import serpapi + +client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"]) +results = client.search( + engine="google", + q="coffee", + location="Austin, Texas", +) +``` + +Search parameter names stay the same. You can pass them by name, as in the new example, or keep them in a dictionary and pass that to `client.search()`. + +## Migration Checklist + +- Remove `google-search-results` from requirements and dependency files. +- Install `serpapi` with `pip install serpapi`. +- Replace `from serpapi import GoogleSearch` with `import serpapi`. +- Replace `GoogleSearch(params).get_dict()` with `serpapi.Client(api_key=...).search(engine=..., q=..., ...)`. +- Keep using the [SerpApi Playground](https://serpapi.com/playground) and the [SerpApi API documentation](https://serpapi.com/search-api) to confirm engine parameters. diff --git a/docs/user_guide/multiprocessing.md b/docs/user_guide/multiprocessing.md new file mode 100644 index 0000000..fd4e041 --- /dev/null +++ b/docs/user_guide/multiprocessing.md @@ -0,0 +1,74 @@ +--- +title: "Multiprocessing" +description: "Run searches in separate Python processes with ProcessPoolExecutor." +--- + +# Multiprocessing + +Multiprocessing runs work in separate Python processes. Use it when you do substantial computation on each search response, need separate process memory, or already use a process pool in your application. [Threads](threading.md) have less startup overhead for searches that mainly wait for network responses. + +## ProcessPoolExecutor Example + +After [setting your API key](getting-started.md#installation), save this example in a Python file and run it from your terminal. `max_workers=4` allows up to four worker processes. Each worker creates its own client and returns a dictionary with selected results. + +```python +import os +import serpapi + +from concurrent.futures import ProcessPoolExecutor, as_completed + + +def search_country(country): + client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) + results = client.search( + engine="google", + q="best coffee beans", + gl=country, + hl="en", + ) + + organic_results = results.get("organic_results", []) + return { + "country": country, + "count": len(organic_results), + "first_title": organic_results[0].get("title") if organic_results else None, + } + + +if __name__ == "__main__": + countries = ["us", "gb", "ca", "au", "in"] + + with ProcessPoolExecutor(max_workers=4) as executor: + futures = [executor.submit(search_country, country) for country in countries] + + for future in as_completed(futures): + print(future.result()) +``` + +Keep `if __name__ == "__main__":` around the code that starts the pool. When a new worker imports the file, this guard prevents it from starting another pool. It is required when workers start this way, as they do by default on macOS and Windows. + +## Practical Guidance + +- Create the `serpapi.Client` inside the worker process. +- Return data that Python can copy between processes, such as dictionaries, lists, strings, and numbers. +- Avoid passing open sessions, response objects, or client instances between processes. +- Set a timeout on each client so a stalled request does not wait indefinitely. +- Use threads if your workers only make searches and read a few fields from the response. + +## Combining Pagination and Processes + +You can submit a different Google `start` offset to each worker. Here is a worker function that fetches one range of results: + +```python +def search_offset(start): + client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) + results = client.search( + engine="google", + q="coffee", + location="Austin, Texas", + start=start, + ) + return results.get("organic_results", []) +``` + +Use the [pagination parameters](pagination.md#pagination-parameters-vary-by-engine) for the engine you are searching. Other engines may require a token from the previous page to request the next one. diff --git a/docs/user_guide/output-formats.md b/docs/user_guide/output-formats.md new file mode 100644 index 0000000..d779b0a --- /dev/null +++ b/docs/user_guide/output-formats.md @@ -0,0 +1,57 @@ +--- +title: "Output Formats" +description: "Choose JSON, Markdown, or HTML results with the output parameter." +--- + +# Output Formats + +Set `output` on `client.search()` to choose the response format. JSON is the default. + +| `output` | Python return type | Use | +| --- | --- | --- | +| `"json"` | `serpapi.SerpResults` | Read individual fields, such as titles and links, and fetch more pages. | +| `"md"` | `str` | Give Markdown search results to AI agents and language models. | +| `"html"` | `str` | Inspect the search results as HTML. | + +## JSON + +JSON stores results as named fields and lists. Use it when your code needs to read individual values, such as a result's title or link. + +After [setting your API key](getting-started.md#installation), create a client and run a search: + +```python +import os +import serpapi + +client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"]) +results = client.search(engine="google", q="coffee", output="json") + +for result in results.get("organic_results", []): + print(result.get("title"), result.get("link")) +``` + +You can leave out `output="json"` because JSON is the default. `SerpResults` behaves like a dictionary and provides `as_dict()`, `next_page()`, and `yield_pages()`. The loop above uses an empty list if `organic_results` is missing. See [Pagination](pagination.md) for examples that fetch more pages. + +## Markdown + +Use `output="md"` for search results formatted with headings, links, and tables: + +```python +markdown = client.search(engine="google", q="coffee", output="md") +print(markdown[:500]) +``` + +The client returns a plain Python string. The example prints its first 500 characters. You can pass the full string to an AI agent as the result of a search tool. See [AI Agents](../ai-agents.md) for integrations and [SerpApi's Markdown output guide](https://serpapi.com/markdown-output) for API details. + +## HTML + +Use `output="html"` to retrieve the HTML of the original search results page: + +```python +html = client.search(engine="google", q="coffee", output="html") +print(html[:500]) +``` + +The client returns HTML as a string. Use JSON when you need to read named result fields or call pagination methods such as `next_page()`. These methods are available on `SerpResults`, and cannot be called on an HTML or Markdown string. + +The same `output` values are supported by `client.search_archive()` when retrieving a saved search. See [Async Search Archive](async-search-archive.md). diff --git a/docs/user_guide/pagination.md b/docs/user_guide/pagination.md new file mode 100644 index 0000000..7999685 --- /dev/null +++ b/docs/user_guide/pagination.md @@ -0,0 +1,86 @@ +--- +title: "Pagination" +description: "Collect additional pages with SerpResults helpers or engine-specific offsets." +--- + +# Pagination + +Pagination means fetching more than one page of search results. If a JSON response includes a next-page link in `serpapi_pagination`, you can read it with `results.next_page_url` or fetch pages with `next_page()` and `yield_pages()`. + +The examples below use a client created as shown in [Client Usage](client-usage.md#create-a-client). + +## Fetch One More Page + +```python +results = client.search( + engine="google", + q="coffee", + location="Austin, Texas", +) + +next_results = results.next_page() + +if next_results: + for item in next_results.get("organic_results", []): + print(item.get("title")) +``` + +`next_page()` returns `None` when the response does not include a next page URL. + +## Iterate Through Pages + +Use `yield_pages()` when you want the current page plus following pages: + +```python +results = client.search( + engine="google", + q="coffee", + location="Austin, Texas", +) + +for page in results.yield_pages(max_pages=5): + for item in page.get("organic_results", []): + print(item.get("position"), item.get("title")) +``` + +Set `max_pages` to limit how many pages your script reads, including the first page. Each additional page requires another request. The loop stops early if there is no next-page link, and results may change between requests. + +## Google `start` Offsets + +Google's `start` parameter tells SerpApi how many results to skip. For example, `start=10` skips the first ten results: + +```python +for start in [0, 10, 20]: + page = client.search( + engine="google", + q="coffee", + location="Austin, Texas", + start=start, + ) + + print("Offset:", start) + for item in page.get("organic_results", []): + print(item.get("title")) +``` + +Set offsets yourself when you need particular result ranges or want to request several ranges at the same time. + +## Store Only What You Need + +To keep less data in memory or storage, copy the fields you need into a list of dictionaries: + +```python +records = [] + +for page in results.yield_pages(max_pages=3): + for item in page.get("organic_results", []): + records.append({ + "title": item.get("title"), + "link": item.get("link"), + "snippet": item.get("snippet"), + }) +``` + +## Pagination Parameters Vary by Engine + +`start` is common for Google Search, but other engines may use different pagination parameters. Check the relevant engine page in the [SerpApi Search API documentation](https://serpapi.com/search-api) or build the request in the [SerpApi Playground](https://serpapi.com/playground). diff --git a/docs/user_guide/parameters-and-engines.md b/docs/user_guide/parameters-and-engines.md new file mode 100644 index 0000000..ffdf39f --- /dev/null +++ b/docs/user_guide/parameters-and-engines.md @@ -0,0 +1,82 @@ +--- +title: "Parameters and Engines" +description: "Choose a search engine and pass its parameters to the Python client." +--- + +# Parameters and Engines + +The `engine` parameter selects which search service to use. The Python client sends your parameters to SerpApi without checking them against a fixed list. You can use a new engine or parameter as soon as SerpApi supports it. + +## Full Engine and Parameter Reference + +Find the parameters for your search in these resources: + +- [SerpApi Search API documentation](https://serpapi.com/search-api) lists supported engines and engine-specific parameters. +- [SerpApi Playground](https://serpapi.com/playground) lets you test a search and copy the exact parameters into Python. +- [Account and Locations](account-and-locations.md) shows how to find valid `location` values from Python. + +The Search API docs include engines such as Google Search, Google Maps, Google Images, Google Shopping, Google Scholar, Google Jobs, Bing, DuckDuckGo, Yahoo, Yandex, Baidu, YouTube, eBay, Walmart, Naver, Apple App Store, Home Depot, and many more. Check the docs for the current full list. + +## Passing Parameters + +Create a client as shown in [Client Usage](client-usage.md#create-a-client). Pass parameters by name, using Python keyword arguments: + +```python +results = client.search( + engine="google", + q="coffee", + location="Austin, Texas", + hl="en", + gl="us", +) +``` + +You can also pass a dictionary when you already have parameters in one: + +```python +params = { + "engine": "google", + "q": "coffee", + "location": "Austin, Texas", + "hl": "en", + "gl": "us", +} +results = client.search(params) +``` + +## Common Google Parameters + +These are common Google Search parameters. Other engines have their own parameter names. + +| Parameter | Purpose | +| --- | --- | +| `engine` | Search engine. Use `google` for Google Search. | +| `q` | Text to search for. | +| `location` | Geographic location for the search. | +| `google_domain` | Google domain, such as `google.com` or `google.co.in`. | +| `gl` | Country code for the search. | +| `hl` | Interface language. | +| `start` | Number of results to skip when fetching another page. | +| `device` | Device type, such as desktop or mobile. | +| `output` | Response format: `json` (default), `html`, or `md`. | +| `async` | Submit a search now and retrieve its results later. | +| `no_cache` | Force SerpApi to fetch fresh results. | +| `zero_trace` | Skip storing the search on SerpApi's servers when your account supports ZeroTrace. | +| `json_restrictor` | Return only selected JSON fields from the API response. | + +For the full supported list and the exact meaning of each parameter, use the [SerpApi Search API documentation](https://serpapi.com/search-api). + +See [Output Formats](output-formats.md) for examples of each response format and its Python return type. + +## Engine-Specific Names + +Different engines use different query parameter names. For example: + +```python +google = client.search(engine="google", q="coffee") +ebay = client.search(engine="ebay", _nkw="coffee grinder") +youtube = client.search(engine="youtube", search_query="coffee brewing") +walmart = client.search(engine="walmart", query="coffee") +``` + +Select the engine in the [SerpApi Playground](https://serpapi.com/playground), try your search, and copy the generated parameters. diff --git a/docs/user_guide/request-options.md b/docs/user_guide/request-options.md new file mode 100644 index 0000000..60c70db --- /dev/null +++ b/docs/user_guide/request-options.md @@ -0,0 +1,137 @@ +--- +title: "Request Options" +description: "Pass requests library options such as timeout, proxies, verify, stream, and cert." +--- + +# Request Options + +Search parameters such as `engine`, `q`, `location`, `hl`, `gl`, `no_cache`, and `json_restrictor` tell SerpApi what to search for and return. Request options such as `timeout` and `proxies` control how your Python program connects to SerpApi through the `requests` library. + +The examples use a client created as shown in [Client Usage](client-usage.md#create-a-client). + +## Supported Request Options + +The client passes these keyword arguments to `requests.Session.request()`: + +| Option | Use | +| --- | --- | +| `timeout` | Set a different timeout, in seconds, for one request. | +| `proxies` | Send the request through HTTP or HTTPS proxies. | +| `verify` | Check the server's TLS certificate, or use a custom certificate authority bundle. | +| `stream` | Set the `requests` streaming option. The client still reads the response before returning results. | +| `cert` | Authenticate the request with a client certificate. | + +These options are supported by `search()`, `search_archive()`, `account()`, and `locations()`. + +## Per-Request Timeout + +Set a default timeout when creating the client: + +```python +client = serpapi.Client(api_key="secret_api_key", timeout=20) +``` + +Override it for a single request: + +```python +results = client.search( + engine="google", + q="coffee", + timeout=5, +) +``` + +`timeout=5` sets the connection and read timeout to five seconds. It is passed to `requests`. See [Errors and Timeouts](errors-and-timeouts.md#connection-errors-and-timeouts) for timeout behavior. + +## Proxies + +To connect through a proxy server, pass a dictionary that maps the request protocol to the proxy URL: + + +```python +results = client.search( + engine="google", + q="coffee", + proxies={ + "https": "http://proxy.example.com:8080", + }, +) +``` + +Keep proxy credentials in environment variables or your secret manager rather than hardcoding them in source files. + +## TLS Verification + +By default, `requests` checks the server's TLS certificate to verify its identity. If your network uses a private certificate authority (CA), pass the path to its trusted certificate bundle: + + +```python +results = client.search( + engine="google", + q="coffee", + verify="/path/to/ca-bundle.pem", +) +``` + +Only disable verification for controlled local debugging: + + +```python +results = client.search( + engine="google", + q="coffee", + verify=False, +) +``` + +Do not use `verify=False` in production code. + +## Client Certificates + +Some servers or proxies require a certificate to identify the client. Pass its file path with `cert`, or pass a `(cert, key)` tuple if the certificate and private key are in separate files: + + +```python +results = client.search( + engine="google", + q="coffee", + cert=("/path/to/client-cert.pem", "/path/to/client-key.pem"), +) +``` + +## Combining Search Parameters and Request Options + +Request options can be used alongside normal SerpApi parameters: + +```python +results = client.search( + engine="google", + q="coffee", + location="Austin, Texas", + hl="en", + gl="us", + json_restrictor="organic_results[].{title, link}", + timeout=10, +) +``` + +The client sends the arguments to two places: + +- `timeout`, `proxies`, `verify`, `stream`, and `cert` are passed to `requests`. +- All remaining keyword arguments are sent to SerpApi as API parameters. + +If you pass a parameter dictionary and keyword arguments together, the search parameters in the keyword arguments update your dictionary: + +```python +params = {"engine": "google", "q": "coffee"} +results = client.search(params, location="Austin, Texas", timeout=10) +``` + +In this example, `location` is added to the SerpApi request parameters, while `timeout` is passed to `requests`. + +When you want to keep the original dictionary unchanged, pass a copy: + +```python +params = {"engine": "google", "q": "coffee"} +results = client.search(params.copy(), location="Austin, Texas") +``` diff --git a/docs/user_guide/threading.md b/docs/user_guide/threading.md new file mode 100644 index 0000000..7b7259b --- /dev/null +++ b/docs/user_guide/threading.md @@ -0,0 +1,74 @@ +--- +title: "Threading" +description: "Run several independent searches at the same time with ThreadPoolExecutor." +--- + +# Threading + +Search requests spend time waiting for network responses. Threads let your program make another request while one is waiting. Use them for independent searches, such as running the same query in several countries. + +## ThreadPoolExecutor Example + +After [setting your API key](getting-started.md#installation), run this example. `max_workers=5` allows up to five searches to run at once. Each submitted task returns a `Future`, an object that holds its eventual result or exception. `as_completed()` gives you each task as it finishes. + +```python +import os +import serpapi + +from concurrent.futures import ThreadPoolExecutor, as_completed + + +API_KEY = os.environ["SERPAPI_KEY"] + + +def search_country(country): + client = serpapi.Client(api_key=API_KEY, timeout=20) + results = client.search( + engine="google", + q="best coffee beans", + gl=country, + hl="en", + ) + return country, results.get("organic_results", []) + + +countries = ["us", "gb", "ca", "au", "in"] + +with ThreadPoolExecutor(max_workers=5) as executor: + futures = [executor.submit(search_country, country) for country in countries] + + for future in as_completed(futures): + country, organic_results = future.result() + first = organic_results[0] if organic_results else {} + print(country, first.get("title")) +``` + +## Practical Guidance + +- Start with a small `max_workers` value. Increase it based on your account's search limits, response times, and how quickly your code can process results. +- Handle each task's exception so the loop can continue processing other results. +- Create a client inside each worker to give it a separate HTTP session. +- Set a timeout so a stalled request does not hold up the batch indefinitely. + +## Handling Per-Request Errors + +`future.result()` raises any exception from the search. Catch it inside the loop to report the failed country and continue: + +```python +with ThreadPoolExecutor(max_workers=5) as executor: + future_to_country = { + executor.submit(search_country, country): country + for country in countries + } + + for future in as_completed(future_to_country): + country = future_to_country[future] + + try: + country, organic_results = future.result() + except serpapi.SerpApiError as exc: + print("Failed:", country, exc) + continue + + print("Finished:", country, len(organic_results)) +``` diff --git a/docs/user_guide/zero-trace.md b/docs/user_guide/zero-trace.md new file mode 100644 index 0000000..c6fcfef --- /dev/null +++ b/docs/user_guide/zero-trace.md @@ -0,0 +1,47 @@ +--- +title: "Zero Trace and Data Retention" +description: "Control search storage and request fresh results with zero_trace and no_cache." +--- + +# Zero Trace and Data Retention + +Use `zero_trace` to control whether SerpApi stores a search, and `no_cache` to request fresh results. Pass these parameters to a client created as shown in [Client Usage](client-usage.md#create-a-client). + +## Zero Trace + +If your account supports ZeroTrace, pass `zero_trace=True` to skip storing search parameters, files, and metadata on SerpApi's servers: + +```python +results = client.search( + engine="google", + q="coffee", + location="Austin, Texas", + zero_trace=True, +) +``` + +Check your plan and the [SerpApi Search API documentation](https://serpapi.com/search-api) for ZeroTrace availability and data retention details. + +## Cache Controls + +SerpApi can reuse a saved response when the search parameters match a recent request. Pass `no_cache=True` to fetch fresh results: + +```python +results = client.search( + engine="google", + q="coffee", + no_cache=True, +) +``` + +`no_cache=true` and `async=true` should not be used together. + +## Choosing Between Modes + +| Need | Parameter | +| --- | --- | +| Fetch fresh results | `no_cache=true` | +| Submit now and fetch later | `async=true` plus `search_archive()` | +| Skip storing the search on SerpApi's servers | `zero_trace=true` when enabled for your account | + +The table uses API parameter syntax. In Python, write `True` with a capital `T`. See [Async Search Archive](async-search-archive.md) for how to pass the `async` parameter. diff --git a/pyproject.toml b/pyproject.toml index d345b89..2c259a9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,12 @@ classifiers = [ [project.optional-dependencies] color = ["pygments"] test = ["pytest"] +docs = [ + "sphinx>=8.2,<9; python_version >= '3.11'", + "myst-parser>=4,<5; python_version >= '3.11'", + "sphinx-design>=0.6,<1; python_version >= '3.11'", + "sphinx-rtd-theme>=3.1,<4; python_version >= '3.11'", +] [project.urls] Homepage = "https://github.com/serpapi/serpapi-python" @@ -50,7 +56,10 @@ Documentation = "https://serpapi-python.readthedocs.io/en/latest/" version = { attr = "serpapi.__version__.__version__" } [tool.setuptools.packages.find] +include = ["serpapi", "serpapi.*"] exclude = ["tests", "tests.*"] [tool.pytest.ini_options] -testpaths = ["tests"] +testpaths = ["tests", "serpapi"] +addopts = "--doctest-modules" +doctest_optionflags = ["ELLIPSIS", "NORMALIZE_WHITESPACE"] diff --git a/scripts/check_docs_revision.py b/scripts/check_docs_revision.py new file mode 100644 index 0000000..d945882 --- /dev/null +++ b/scripts/check_docs_revision.py @@ -0,0 +1,53 @@ +import json +import os +import re +import subprocess +import sys +import time + + +CONTEXT_VARIABLE = "DOCS_CI_REVISION" + + +def check_revision(environ, commit, now=None): + if environ.get("READTHEDOCS_VERSION_TYPE") == "external": + return + try: + context = json.loads(environ[CONTEXT_VARIABLE]) + expected = context["commit"] + versions = context["versions"] + expires = context["expires_at"] + if not re.fullmatch(r"[0-9a-f]{40}", expected): + raise ValueError + if not isinstance(versions, list) or not versions: + raise ValueError + if not isinstance(expires, (int, float)): + raise ValueError + except (KeyError, TypeError, ValueError): + raise RuntimeError( + "No valid CI revision was supplied. Publish through the Documentation workflow." + ) from None + if (time.time() if now is None else now) >= expires: + raise RuntimeError("The CI revision expired. Rerun the Documentation workflow.") + name = environ.get("READTHEDOCS_VERSION_NAME") or environ.get("READTHEDOCS_VERSION") + if name not in versions: + raise RuntimeError("CI did not request publication of this documentation version.") + if commit != expected: + raise RuntimeError( + "RTD checked out a different commit than the one tested by CI. " + "Rerun the Documentation workflow on the current branch or tag." + ) + + +def main(): + commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip() + check_revision(os.environ, commit) + print("Documentation revision check passed.") + + +if __name__ == "__main__": + try: + main() + except RuntimeError as exc: + print(str(exc), file=sys.stderr) + sys.exit(1) diff --git a/scripts/publish_docs.py b/scripts/publish_docs.py new file mode 100644 index 0000000..7e6e489 --- /dev/null +++ b/scripts/publish_docs.py @@ -0,0 +1,207 @@ +import json +import os +import re +import subprocess +import sys +import time +from urllib.error import HTTPError, URLError +from urllib.parse import quote, urlencode, urljoin +from urllib.request import HTTPRedirectHandler, Request, build_opener + +from scripts.check_docs_revision import CONTEXT_VARIABLE + + +PROJECT = "serpapi-python" +API_ROOT = "https://app.readthedocs.org/api/v3/" +POLL_SECONDS = 10 +BUILD_TIMEOUT = 1200 + + +class APIError(RuntimeError): + def __init__(self, status): + self.status = status + super().__init__(f"Read the Docs API returned HTTP {status}.") + + +class NoRedirect(HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None + + +class ReadTheDocs: + def __init__(self, token): + self.token = token + self.base = f"{API_ROOT}projects/{PROJECT}/" + self.open = build_opener(NoRedirect()).open + + def request(self, method, path, data=None): + url = urljoin(self.base, path) + if not url.startswith(self.base): + raise RuntimeError("Refusing to send the RTD token outside this project's API.") + body = None if data is None else json.dumps(data).encode() + request = Request(url, data=body, method=method, headers={ + "Authorization": f"Token {self.token}", + "Content-Type": "application/json", + "User-Agent": "serpapi-python-docs-publisher", + }) + try: + with self.open(request, timeout=30) as response: + payload = response.read() + except HTTPError as exc: + raise APIError(exc.code) from None + except (URLError, TimeoutError): + raise RuntimeError("Could not reach the Read the Docs API.") from None + try: + return json.loads(payload) if payload else None + except ValueError: + raise RuntimeError("Read the Docs returned an invalid API response.") from None + + def items(self, path): + while path: + page = self.request("GET", path) + yield from page["results"] + path = page.get("next") + + +def wait_for(check, message, timeout=BUILD_TIMEOUT): + deadline = time.monotonic() + timeout + while True: + result = check() + if result: + return result + if time.monotonic() >= deadline: + raise RuntimeError(message) + time.sleep(POLL_SECONDS) + + +def version_path(slug): + return f"versions/{quote(slug, safe='')}/" + + +def find_version(api, ref): + if ref == "refs/heads/master": + return api.request("GET", version_path("latest")) + query = urlencode({"type": "tag", "verbose_name": ref[len("refs/tags/"):]}) + versions = list(api.items(f"versions/?{query}")) + if len(versions) > 1: + raise RuntimeError("More than one RTD version matches the release tag.") + return versions[0] if versions else None + + +def wait_for_build(api, build, commit): + build_id = build["id"] + url = f"https://app.readthedocs.org/projects/{PROJECT}/builds/{build_id}/" + print(f"Waiting for RTD build: {url}", flush=True) + + def finished(): + current = api.request("GET", f"builds/{build_id}/") + state = current["state"]["code"] + if state not in ("finished", "cancelled"): + return None + if not current.get("success"): + raise RuntimeError(f"RTD build failed or was cancelled. See {url}") + if current.get("commit") != commit: + raise RuntimeError(f"RTD built a different commit. See {url}") + return current + + return wait_for(finished, f"Timed out waiting for RTD. See {url}") + + +def newest_build(api, version, after): + for build in api.items("builds/"): + if build["id"] <= after: + break + if build["version"] == version: + return build + return None + + +def publish_version(api, version, commit, after=None): + path = version_path(version["slug"]) + # Syncing versions can queue stable and release builds without an automation rule. + build = newest_build(api, version["slug"], after) if after is not None else None + if build: + return wait_for_build(api, build, commit) + if not version["active"]: + previous = api.request("GET", "builds/?limit=1")["results"] + last_id = previous[0]["id"] if previous else 0 + # Activating a version already queues a build; do not queue a second one. + api.request("PATCH", path, {"active": True, "hidden": False}) + + def activated_build(): + return newest_build(api, version["slug"], last_id) + + build = wait_for(activated_build, "RTD did not queue the activated version.", 120) + else: + build = api.request("POST", path + "builds/")["build"] + return wait_for_build(api, build, commit) + + +def clear_context(api): + for variable in list(api.items("environmentvariables/")): + if variable["name"] == CONTEXT_VARIABLE: + api.request("DELETE", f"environmentvariables/{variable['pk']}/") + + +def publish(api, ref, commit): + if ref != "refs/heads/master" and not ref.startswith("refs/tags/v"): + raise RuntimeError("Only master and v-prefixed release tags can publish documentation.") + if not re.fullmatch(r"[0-9a-f]{40}", commit): + raise RuntimeError("Expected a full Git commit hash.") + project = api.request("GET", "") + if project["default_branch"] != "master": + raise RuntimeError("Set the RTD project's Default branch to master first.") + + # Serialize in Actions, and let any previous RTD build finish before changing its context. + wait_for( + lambda: not api.request("GET", "builds/?running=true")["results"], + "Existing RTD builds did not finish. Check the RTD dashboard before retrying.", + ) + clear_context(api) + previous = api.request("GET", "builds/?limit=1")["results"] + last_id = previous[0]["id"] if previous else 0 + versions = ["latest"] if ref == "refs/heads/master" else [ref[len("refs/tags/"):], "stable"] + context = { + "commit": commit, + "versions": versions, + "expires_at": time.time() + 3600, + } + variable = api.request("POST", "environmentvariables/", { + "name": CONTEXT_VARIABLE, "value": json.dumps(context), "public": False, + }) + try: + api.request("POST", "sync-versions/") + version = wait_for(lambda: find_version(api, ref), "RTD did not discover the Git tag.", 120) + publish_version(api, version, commit, after=last_id) + if ref.startswith("refs/tags/"): + try: + stable = api.request("GET", version_path("stable")) + except APIError as exc: + if exc.status != 404: + raise + stable = None + if stable and stable.get("ref") == ref[len("refs/tags/"):]: + publish_version(api, stable, commit, after=last_id) + print(f"Published documentation for {ref} at {commit}.", flush=True) + finally: + api.request("DELETE", f"environmentvariables/{variable['pk']}/") + + +def main(): + token = os.environ.get("RTD_API_TOKEN") + if not token: + raise RuntimeError("Add RTD_API_TOKEN to the docs GitHub environment before publishing.") + if os.environ.get("GITHUB_EVENT_NAME") not in ("push", "workflow_dispatch"): + raise RuntimeError("PR runs cannot publish documentation.") + commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip() + if commit != os.environ.get("GITHUB_SHA"): + raise RuntimeError("The publishing checkout does not match the commit tested by CI.") + publish(ReadTheDocs(token), os.environ.get("GITHUB_REF", ""), commit) + + +if __name__ == "__main__": + try: + main() + except RuntimeError as exc: + print(str(exc), file=sys.stderr) + sys.exit(1) diff --git a/serpapi/core.py b/serpapi/core.py index 8622a82..f24e089 100644 --- a/serpapi/core.py +++ b/serpapi/core.py @@ -7,23 +7,13 @@ class Client(HTTPClient): - """A class that handles API requests to SerpApi in a user–friendly manner. + """A class that handles API requests to SerpApi in a user-friendly manner. - :param api_key: The API Key to use for SerpApi.com. - - Please provide ``api_key`` when instantiating this class. We recommend storing this in an environment variable, like so: - - .. code-block:: bash - - $ export SERPAPI_KEY=YOUR_API_KEY - - .. code-block:: python - - import os - import serpapi - - serpapi = serpapi.Client(api_key=os.environ["SERPAPI_KEY"]) + Store your API key in an environment variable, then create a client with + ``serpapi.Client(api_key=os.environ["SERPAPI_KEY"])``. + :param api_key: The API Key to use for SerpApi.com. + :param timeout: The default timeout to use for requests. """ DASHBOARD_URL = "https://serpapi.com/dashboard" @@ -35,30 +25,16 @@ def __repr__(self): return "" def search(self, params: dict = None, **kwargs): - """Fetch a page of results from SerpApi. Returns a :class:`SerpResults ` object for JSON responses, or unicode text for HTML and Markdown responses. - - The following three calls are equivalent: + """Fetch a page of results from SerpApi. - .. code-block:: python + Returns a ``serpapi.SerpResults`` object for JSON responses, or text + when ``output="html"`` or ``output="md"`` is requested. Prefer passing SerpApi engine + parameters as keyword arguments. A parameter dictionary is also accepted + when your code already has parameters in a mapping. - >>> s = serpapi.search(q="Coffee", location="Austin, Texas, United States") - .. code-block:: python - - >>> params = {"q": "Coffee", "location": "Austin, Texas, United States"} - >>> s = serpapi.search(**params) - - .. code-block:: python - - >>> params = {"q": "Coffee", "location": "Austin, Texas, United States"} - >>> s = serpapi.search(params) - - - :param q: typically, this is the parameter for the search engine query. - :param engine: the search engine to use. Defaults to ``google``. - :param output: the output format desired (``html``, ``json``, or ``md``). Defaults to ``json``. - :param api_key: the API Key to use for SerpApi.com. - :param **: any additional parameters to pass to the API. + :param params: Optional mapping of SerpApi search parameters such as ``engine``, ``q``, ``location``, and ``output``. + :param kwargs: Additional SerpApi parameters or request options. ``timeout``, ``proxies``, ``verify``, ``stream``, and ``cert`` are passed to the underlying HTTP request. **Learn more**: https://serpapi.com/search-api @@ -82,10 +58,8 @@ def search(self, params: dict = None, **kwargs): def search_archive(self, params: dict = None, **kwargs): """Get a result from the SerpApi Search Archive API. - :param search_id: the Search ID of the search to retrieve from the archive. - :param api_key: the API Key to use for SerpApi.com. - :param output: the output format desired (``html``, ``json``, or ``md``). Defaults to ``json``. - :param **: any additional parameters to pass to the API. + :param params: Archive parameters. Must include ``search_id``. ``output`` accepts ``json`` (default), ``html``, or ``md``. + :param kwargs: Additional archive parameters or request options. ``timeout``, ``proxies``, ``verify``, ``stream``, and ``cert`` are passed to the underlying HTTP request. **Learn more**: https://serpapi.com/search-archive-api """ @@ -162,9 +136,8 @@ def locations(self, params: dict = None, **kwargs): """Get a list of supported Google locations. - :param q: restricts your search to locations that contain the supplied string. - :param limit: limits the number of locations returned. - :param **: any additional parameters to pass to the API. + :param params: Location API parameters such as ``q`` and ``limit``. + :param kwargs: Additional location parameters or request options. ``timeout``, ``proxies``, ``verify``, ``stream``, and ``cert`` are passed to the underlying HTTP request. **Learn more**: https://serpapi.com/locations-api """ @@ -192,8 +165,8 @@ def locations(self, params: dict = None, **kwargs): def account(self, params: dict = None, **kwargs): """Get SerpApi account information. - :param api_key: the API Key to use for SerpApi.com. - :param **: any additional parameters to pass to the API. + :param params: Account API parameters. + :param kwargs: Additional account parameters or request options. ``timeout``, ``proxies``, ``verify``, ``stream``, and ``cert`` are passed to the underlying HTTP request. **Learn more**: https://serpapi.com/account-api """ diff --git a/serpapi/models.py b/serpapi/models.py index 000b2a3..3c825a8 100644 --- a/serpapi/models.py +++ b/serpapi/models.py @@ -10,13 +10,6 @@ class SerpResults(UserDict): """A dictionary-like object that represents the results of a SerpApi request. - .. code-block:: python - - >>> search = serpapi.search(q="Coffee", location="Austin, Texas, United States") - - >>> print(search["search_metadata"].keys()) - dict_keys(['id', 'status', 'json_endpoint', 'created_at', 'processed_at', 'google_url', 'raw_html_file', 'total_time_taken']) - An instance of this class is returned if the response is a valid JSON object. It can be used like a dictionary, but also has some additional methods. """ @@ -40,7 +33,10 @@ def __repr__(self): def as_dict(self): """Returns the data as a standard Python dictionary. - This can be useful when using ``json.dumps(search), for example.""" + + This can be useful when passing results to libraries that expect a + plain ``dict``. + """ return self.data.copy() @@ -85,7 +81,7 @@ def yield_pages(self, max_pages=1_000): def from_http_response(cls, r, *, client=None): """Construct a SerpResults object from an HTTP response. - :param assert_200: if ``True`` (default), raise an exception if the status code is not 200. + :param r: The HTTP response to parse. :param client: the Client instance which was used to send this request. An instance of this class is returned if the response is a valid JSON object. diff --git a/tests/conftest.py b/tests/conftest.py index e494aaf..fb93fcf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,6 +7,21 @@ os.environ["CI"] = "1" +def pytest_addoption(parser): + parser.addoption( + "--require-docs-key", + action="store_true", + help="Fail instead of skipping live docs examples when the API key is missing", + ) + + +def pytest_sessionstart(session): + if session.config.getoption("--require-docs-key") and not ( + os.environ.get("SERPAPI_KEY") or os.environ.get("API_KEY") + ): + raise pytest.UsageError("Live docs checks require SERPAPI_KEY or API_KEY") + + @pytest.fixture def api_key(): return os.environ["API_KEY"] diff --git a/tests/docs_example_support.py b/tests/docs_example_support.py new file mode 100644 index 0000000..7d61d72 --- /dev/null +++ b/tests/docs_example_support.py @@ -0,0 +1,161 @@ +import json +import os +from pathlib import Path +import re +import shutil +import signal +import subprocess +import sys +import threading +from urllib.parse import parse_qs, urlsplit + +import serpapi +from serpapi.http import HTTPClient + + +ROOT = Path(__file__).resolve().parents[1] +PYTHON_BLOCK_RE = re.compile( + r"(?P\s*)?" + r"```python\r?\n(?P.*?)```", + re.DOTALL, +) +PAGE_TIMEOUT = 300 + + +def python_blocks(path): + return [match.groupdict() for match in PYTHON_BLOCK_RE.finditer(path.read_text())] + + +def docs_pages(): + paths = [ROOT / "README.md", *sorted((ROOT / "docs").rglob("*.md"))] + return [path for path in paths if "_build" not in path.parts and python_blocks(path)] + + +def redact(text, api_key=None): + for key in (api_key, os.environ.get("SERPAPI_KEY"), os.environ.get("API_KEY")): + if key: + text = text.replace(key, "[REDACTED]") + return text + + +def validate_response(response, path, params): + if not 200 <= response.status_code < 300: + raise AssertionError(f"HTTP {response.status_code}") + + content_type = response.headers.get("Content-Type", "").lower() + if "json" in content_type: + payload = response.json() + if not isinstance(payload, (dict, list)) or not payload: + raise AssertionError("Expected a nonempty JSON object or list") + if isinstance(payload, dict): + if payload.get("error"): + raise AssertionError(str(payload["error"])) + if payload.get("search_metadata", {}).get("status") == "Error": + raise AssertionError("Search metadata reports an error") + if path == "/image" and not payload.get("image_id"): + raise AssertionError("Image upload did not return image_id") + + if path == "/search" or path.startswith("/searches/"): + result = serpapi.SerpResults.from_http_response(response) + output = params.get("output", "json") + if output in ("md", "html"): + if not isinstance(result, str) or not result.strip(): + raise AssertionError(f"output={output} did not return a nonempty string") + elif not isinstance(result, serpapi.SerpResults): + raise AssertionError("JSON search did not return SerpResults") + + +def install_http_checks(): + original_request = HTTPClient.request + report_path = Path(os.environ["DOCS_EXAMPLE_REPORT_DIR"]) / f"{os.getpid()}.jsonl" + lock = threading.Lock() + + def checked_request(self, method, path, params, **kwargs): + url = urlsplit(path) + query = {key: values[-1] for key, values in parse_qs(url.query).items()} + query.update(params) + record = {"path": url.path, "engine": query.get("engine"), "ok": False} + if not self.timeout and not kwargs.get("timeout"): + kwargs["timeout"] = 30 + try: + response = original_request(self, method, path, params, **kwargs) + validate_response(response, url.path, query) + record["ok"] = True + return response + except Exception as exc: + record["error"] = redact(f"{type(exc).__name__}: {exc}") + raise + finally: + with lock, report_path.open("a") as report: + report.write(json.dumps(record) + "\n") + + HTTPClient.request = checked_request + + +def script_for_page(path): + parts = [ + "import os\nimport serpapi\n" + "from tests.docs_example_support import install_http_checks\n" + "install_http_checks()\n" + 'client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=30)\n' + ] + for number, block in enumerate(python_blocks(path), start=1): + if block["skip"]: + if not block["reason"].strip(): + raise RuntimeError(f"Python block {number} needs a docs-test skip reason") + continue + code = block["code"] + for placeholder in ('"secret_api_key"', "'secret_api_key'"): + code = code.replace(placeholder, 'os.environ["SERPAPI_KEY"]') + parts.append(f"# Python block {number}\n{code}") + return "\n\n".join(parts) + + +def run_page(path, workdir, api_key, timeout=PAGE_TIMEOUT): + workdir.mkdir(parents=True, exist_ok=True) + report_dir = workdir / "requests" + report_dir.mkdir() + script = workdir / "example.py" + script.write_text(script_for_page(path)) + shutil.copyfile(ROOT / "assets" / "serpapi-icon.png", workdir / "image.png") + env = os.environ.copy() + env.update({ + "SERPAPI_KEY": api_key, + "API_KEY": api_key, + "DOCS_EXAMPLE_REPORT_DIR": str(report_dir), + "PYTHONPATH": os.pathsep.join([str(ROOT / "tests"), str(ROOT)]), + }) + with subprocess.Popen( + [sys.executable, str(script)], + cwd=workdir, + env=env, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + start_new_session=os.name == "posix", + ) as process: + try: + _, stderr = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + if os.name == "posix": + os.killpg(process.pid, signal.SIGKILL) + else: + process.kill() + process.communicate() + raise RuntimeError(f"Example exceeded the {timeout}-second page limit") from None + + records = [ + json.loads(line) + for report in sorted(report_dir.glob("*.jsonl")) + for line in report.read_text().splitlines() + ] + errors = [record for record in records if not record["ok"]] + if process.returncode or errors: + details = "\n".join( + f"{record['engine'] or record['path']}: {record['error']}" + for record in errors + ) + raise RuntimeError(redact(f"{details}\n{stderr}", api_key).strip()) + if not records: + raise RuntimeError("Example finished without making a SerpApi request") + return records diff --git a/tests/test_docs_example_runner.py b/tests/test_docs_example_runner.py new file mode 100644 index 0000000..507cddd --- /dev/null +++ b/tests/test_docs_example_runner.py @@ -0,0 +1,124 @@ +import os +import subprocess +import sys +from textwrap import dedent + +import pytest +import requests + +from tests.docs_example_support import ROOT, run_page, validate_response + + +MOCK_REQUEST = """ +import json +import requests + +def fake_request(self, **kwargs): + response = requests.Response() + response.status_code = 200 + response.headers['Content-Type'] = 'application/json' + response._content = json.dumps({'organic_results': [{'title': 'Coffee'}]}).encode() + return response + +requests.Session.request = fake_request +""" + + +def markdown_page(tmp_path, code): + path = tmp_path / "page.md" + path.write_text("```python\n" + dedent(code) + "\n```\n") + return path + + +def test_doc_runner_executes_main_guard_and_spawned_workers(tmp_path): + code = MOCK_REQUEST + """ +from concurrent.futures import ProcessPoolExecutor +import multiprocessing + +def search_worker(query): + results = client.search(q=query) + return results['organic_results'][0]['title'] + +if __name__ == '__main__': + with ProcessPoolExecutor(max_workers=2, mp_context=multiprocessing.get_context('spawn')) as pool: + assert list(pool.map(search_worker, ['one', 'two'])) == ['Coffee', 'Coffee'] +""" + page = markdown_page(tmp_path, code) + records = run_page(page, tmp_path / "run", "test-key") + assert len(records) == 2 + assert all(record["ok"] for record in records) + + +def test_doc_runner_preserves_block_state_and_skips_marked_examples(tmp_path): + page = markdown_page(tmp_path, MOCK_REQUEST + "\nquery = 'coffee'\n") + with page.open("a") as source: + source.write(""" + +```python +raise RuntimeError('This block must not run') +``` +```python +from pathlib import Path +assert Path('image.png').read_bytes().startswith(b'\\x89PNG') +client = serpapi.Client(api_key='secret_api_key') +assert client.api_key == os.environ['SERPAPI_KEY'] +results = client.search(q=query) +assert results['organic_results'][0]['title'] == 'Coffee' +``` +""") + assert len(run_page(page, tmp_path / "run", "test-key")) == 1 + assert "test-key" not in (tmp_path / "run" / "example.py").read_text() + + +def test_doc_runner_fails_on_caught_api_errors_and_redacts_key(tmp_path): + code = MOCK_REQUEST.replace( + "{'organic_results': [{'title': 'Coffee'}]}", + "{'error': os.environ['SERPAPI_KEY']}", + ) + """ +try: + client.search(q='coffee') +except AssertionError: + pass +""" + page = markdown_page(tmp_path, code) + with pytest.raises(RuntimeError) as failure: + run_page(page, tmp_path / "run", "private-test-key") + assert "[REDACTED]" in str(failure.value) + assert "private-test-key" not in str(failure.value) + for report in (tmp_path / "run" / "requests").glob("*.jsonl"): + assert "private-test-key" not in report.read_text() + + +def test_doc_runner_rejects_examples_without_requests(tmp_path): + page = markdown_page(tmp_path, "if __name__ == 'docs_examples':\n client.search(q='coffee')") + with pytest.raises(RuntimeError, match="without making a SerpApi request"): + run_page(page, tmp_path / "run", "test-key") + + +def test_doc_runner_stops_a_stalled_example(tmp_path): + page = markdown_page(tmp_path, "import time\ntime.sleep(60)") + with pytest.raises(RuntimeError, match="page limit"): + run_page(page, tmp_path / "run", "test-key", timeout=1) + + +@pytest.mark.parametrize("output", ["md", "html"]) +def test_doc_response_check_rejects_json_for_text_output(output): + response = requests.Response() + response.status_code = 200 + response.headers["Content-Type"] = "application/json" + response._content = b'{"organic_results": [{"title": "Coffee"}]}' + with pytest.raises(AssertionError, match="nonempty string"): + validate_response(response, "/search", {"output": output}) + + +def test_doc_gate_fails_without_a_key(): + env = os.environ.copy() + env.pop("SERPAPI_KEY", None) + env.pop("API_KEY", None) + result = subprocess.run( + [sys.executable, "-m", "pytest", "tests/test_docs_examples.py", + "--require-docs-key", "--collect-only", "-q"], + cwd=ROOT, env=env, capture_output=True, text=True, + ) + assert result.returncode != 0 + assert "Live docs checks require SERPAPI_KEY or API_KEY" in result.stderr diff --git a/tests/test_docs_examples.py b/tests/test_docs_examples.py new file mode 100644 index 0000000..3d7494f --- /dev/null +++ b/tests/test_docs_examples.py @@ -0,0 +1,26 @@ +import os + +import pytest + +from tests.docs_example_support import ROOT, docs_pages, python_blocks, run_page + + +PAGES = docs_pages() + + +@pytest.mark.parametrize("path", PAGES, ids=lambda path: path.relative_to(ROOT).as_posix()) +def test_documentation_python_blocks_compile(path): + for number, block in enumerate(python_blocks(path), start=1): + compile(block["code"], f"{path}::python-block-{number}", "exec") + + +@pytest.mark.parametrize("path", PAGES, ids=lambda path: path.relative_to(ROOT).as_posix()) +def test_documentation_examples_live(path, tmp_path): + api_key = os.environ.get("SERPAPI_KEY") or os.environ.get("API_KEY") + if not api_key: + pytest.skip("Set SERPAPI_KEY or API_KEY to run live documentation examples") + + try: + run_page(path, tmp_path, api_key) + except RuntimeError as exc: + pytest.fail(str(exc), pytrace=False) diff --git a/tests/test_docs_publishing.py b/tests/test_docs_publishing.py new file mode 100644 index 0000000..79f70df --- /dev/null +++ b/tests/test_docs_publishing.py @@ -0,0 +1,288 @@ +import io +import json +from collections import deque +from pathlib import Path +import subprocess +from types import SimpleNamespace +from urllib.error import HTTPError + +import pytest + +from scripts import check_docs_revision as guard +from scripts import publish_docs as publisher + + +COMMIT = "a" * 40 +TOKEN = "private-test-token" + + +def ci_environment(**context_changes): + context = {"commit": COMMIT, "versions": ["latest"], "expires_at": 200} + context.update(context_changes) + return { + "READTHEDOCS_VERSION": "latest", + "READTHEDOCS_VERSION_NAME": "latest", + "READTHEDOCS_VERSION_TYPE": "branch", + guard.CONTEXT_VARIABLE: json.dumps(context), + } + + +def test_revision_guard_accepts_only_the_tested_commit(): + guard.check_revision(ci_environment(), COMMIT, now=100) + with pytest.raises(RuntimeError, match="different commit"): + guard.check_revision(ci_environment(), "b" * 40, now=100) + + +@pytest.mark.parametrize("context", [None, "not json", "null", "[]", "{}"]) +def test_revision_guard_rejects_missing_or_malformed_context(context): + env = ci_environment() + if context is None: + env.pop(guard.CONTEXT_VARIABLE) + else: + env[guard.CONTEXT_VARIABLE] = context + with pytest.raises(RuntimeError, match="No valid CI revision"): + guard.check_revision(env, COMMIT, now=100) + + +def test_revision_guard_rejects_expired_and_other_versions(): + with pytest.raises(RuntimeError, match="expired"): + guard.check_revision(ci_environment(expires_at=100), COMMIT, now=100) + with pytest.raises(RuntimeError, match="did not request"): + guard.check_revision(ci_environment(versions=["v1.2.3", "stable"]), COMMIT, now=100) + + +def test_revision_guard_uses_tag_name_instead_of_normalized_slug(): + env = ci_environment(versions=["v1.2.3", "stable"]) + env.update(READTHEDOCS_VERSION="v123", READTHEDOCS_VERSION_NAME="v1.2.3", + READTHEDOCS_VERSION_TYPE="tag") + guard.check_revision(env, COMMIT, now=100) + env["READTHEDOCS_VERSION_NAME"] = "stable" + guard.check_revision(env, COMMIT, now=100) + env["READTHEDOCS_VERSION"] = "stable" + env["READTHEDOCS_VERSION_NAME"] = "v9.0.0" + with pytest.raises(RuntimeError, match="did not request"): + guard.check_revision(env, COMMIT, now=100) + + +def test_revision_guard_falls_back_to_slug_when_name_is_missing(): + env = ci_environment() + env.pop("READTHEDOCS_VERSION_NAME") + guard.check_revision(env, COMMIT, now=100) + + +def test_pr_preview_needs_no_ci_context(): + guard.check_revision({"READTHEDOCS_VERSION_TYPE": "external"}, COMMIT) + + +def page(*items, next=None): + return {"results": list(items), "next": next} + + +def build(build_id=10, **changes): + result = { + "id": build_id, "version": "latest", "commit": COMMIT, + "state": {"code": "finished"}, "success": True, + } + result.update(changes) + return result + + +class HTTPResponses: + # Route requests by endpoint; successive responses model asynchronous changes. + def __init__(self): + self.routes = {} + self.requests = [] + + def respond(self, method, path, *responses): + self.routes[method, path] = deque(responses) + + def open(self, request, timeout): + assert request.get_header("Authorization") == f"Token {TOKEN}" + assert request.get_header("Content-type") == "application/json" + assert TOKEN not in request.full_url + assert timeout > 0 + base = f"{publisher.API_ROOT}projects/{publisher.PROJECT}/" + assert request.full_url.startswith(base) + path = request.full_url[len(base):] + data = json.loads(request.data) if request.data is not None else None + self.requests.append((request.get_method(), path, data)) + responses = self.routes[request.get_method(), path] + response = responses.popleft() if len(responses) > 1 else responses[0] + if isinstance(response, Exception): + raise response + if not isinstance(response, bytes): + response = b"" if response is None else json.dumps(response).encode() + return io.BytesIO(response) + + def sent(self, method, path): + return [data for verb, endpoint, data in self.requests + if (verb, endpoint) == (method, path)] + + +@pytest.fixture +def clock(monkeypatch): + clock = SimpleNamespace(elapsed=0) + + def sleep(seconds): + clock.elapsed += seconds + + monkeypatch.setattr(publisher, "time", SimpleNamespace( + time=lambda: 100, monotonic=lambda: clock.elapsed, sleep=sleep, + )) + return clock + + +@pytest.fixture +def rtd(monkeypatch, clock): + rtd = HTTPResponses() + rtd.respond("GET", "", {"default_branch": "master"}) + rtd.respond("GET", "builds/?running=true", page()) + rtd.respond("GET", "environmentvariables/", page({"name": "UNRELATED", "pk": 1})) + rtd.respond("GET", "builds/?limit=1", page(build(9))) + rtd.respond("POST", "environmentvariables/", {"pk": 2}) + rtd.respond("POST", "sync-versions/", {}) + rtd.respond("GET", "versions/latest/", {"slug": "latest", "active": True}) + rtd.respond("GET", "builds/", page(build(9))) + rtd.respond("POST", "versions/latest/builds/", {"build": build()}) + rtd.respond("GET", "builds/10/", build()) + rtd.respond("DELETE", "environmentvariables/2/", None) + monkeypatch.setattr(publisher, "build_opener", lambda *handlers: SimpleNamespace(open=rtd.open)) + rtd.api = publisher.ReadTheDocs(TOKEN) + return rtd + + +@pytest.fixture +def github_run(monkeypatch, rtd): + monkeypatch.chdir(Path(__file__).resolve().parents[1]) + commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip() + for name, value in { + "RTD_API_TOKEN": TOKEN, "GITHUB_EVENT_NAME": "push", + "GITHUB_SHA": commit, "GITHUB_REF": "refs/heads/master", + }.items(): + monkeypatch.setenv(name, value) + rtd.respond("GET", "builds/10/", build(commit=commit)) + return commit + + +@pytest.mark.parametrize("event", ["push", "workflow_dispatch"]) +def test_main_publishes_the_checkout_with_scoped_context_and_cleanup(monkeypatch, rtd, github_run, event): + monkeypatch.setenv("GITHUB_EVENT_NAME", event) + publisher.main() + creation, = rtd.sent("POST", "environmentvariables/") + assert creation["public"] is False + assert creation["name"] == guard.CONTEXT_VARIABLE + context = json.loads(creation["value"]) + assert context["commit"] == github_run + assert context["versions"] == ["latest"] + assert context["expires_at"] > publisher.time.time() + assert rtd.sent("DELETE", "environmentvariables/2/") == [None] + actions = [(method, path) for method, path, _ in rtd.requests] + assert (actions.index(("POST", "environmentvariables/")) + < actions.index(("POST", "sync-versions/")) + < actions.index(("POST", "versions/latest/builds/")) + < actions.index(("DELETE", "environmentvariables/2/"))) + + +@pytest.mark.parametrize("name, value, message", [ + ("RTD_API_TOKEN", None, "Add RTD_API_TOKEN"), + ("GITHUB_EVENT_NAME", "pull_request", "PR runs cannot publish"), + ("GITHUB_SHA", "0" * 40, "checkout does not match"), +]) +def test_main_rejects_invalid_ci_environment_before_contacting_rtd(monkeypatch, rtd, github_run, name, value, message): + if value is None: + monkeypatch.delenv(name) + else: + monkeypatch.setenv(name, value) + with pytest.raises(RuntimeError, match=message): + publisher.main() + assert not rtd.requests + + +@pytest.mark.parametrize("stable_ref", ["v1.2.3", "v2.0.0", None]) +def test_release_discovers_and_activates_tag_then_publishes_matching_stable(rtd, clock, stable_ref): + tag = {"slug": "v123", "active": False} + rtd.respond("GET", "versions/?type=tag&verbose_name=v1.2.3", page(), page(tag)) + rtd.respond("PATCH", "versions/v123/", None) + rtd.respond("GET", "builds/", page(build(9)), page(build(9)), + page(build(version="v123"))) + rtd.respond("GET", "builds/10/", build(state={"code": "building"}, success=None), + build(version="v123")) + stable = ({"slug": "stable", "active": True, "ref": stable_ref} + if stable_ref else HTTPError(rtd.api.base + "versions/stable/", 404, "Not found", {}, None)) + rtd.respond("GET", "versions/stable/", stable) + rtd.respond("POST", "versions/stable/builds/", {"build": build(11, version="stable")}) + rtd.respond("GET", "builds/11/", build(11, version="stable")) + + publisher.publish(rtd.api, "refs/tags/v1.2.3", COMMIT) + + assert clock.elapsed > 0 + assert rtd.sent("PATCH", "versions/v123/") == [{"active": True, "hidden": False}] + assert not rtd.sent("POST", "versions/v123/builds/") + assert len(rtd.sent("POST", "versions/stable/builds/")) == (stable_ref == "v1.2.3") + creation, = rtd.sent("POST", "environmentvariables/") + assert json.loads(creation["value"])["versions"] == ["v1.2.3", "stable"] + assert rtd.sent("DELETE", "environmentvariables/2/") == [None] + + +def test_publisher_reuses_matching_build_started_by_sync(rtd): + rtd.respond("GET", "builds/", page(build(11, version="v123"), build(10, version="stable"))) + rtd.respond("GET", "builds/10/", build(version="stable")) + result = publisher.publish_version(rtd.api, {"slug": "stable", "active": True}, COMMIT, after=9) + assert result["id"] == 10 + assert all(method == "GET" for method, _, _ in rtd.requests) + + +@pytest.mark.parametrize("outcome, message", [ + (build(success=False), "failed or was cancelled"), + (build(state={"code": "cancelled"}, success=None), "failed or was cancelled"), + (build(commit="b" * 40), "different commit"), + (build(state={"code": "building"}, success=None), "Timed out waiting"), +]) +def test_failed_or_stalled_publication_removes_context(rtd, outcome, message): + rtd.respond("GET", "builds/10/", outcome) + with pytest.raises(RuntimeError, match=message): + publisher.publish(rtd.api, "refs/heads/master", COMMIT) + assert len(rtd.sent("POST", "environmentvariables/")) == 1 + assert rtd.sent("DELETE", "environmentvariables/2/") == [None] + + +def test_publisher_waits_for_existing_builds_before_changing_context(rtd): + rtd.respond("GET", "builds/?running=true", page(build()), page()) + publisher.publish(rtd.api, "refs/heads/master", COMMIT) + actions = [(method, path) for method, path, _ in rtd.requests] + polls = [index for index, action in enumerate(actions) if action == ("GET", "builds/?running=true")] + assert len(polls) >= 2 + assert max(polls) < actions.index(("POST", "environmentvariables/")) + + +def test_context_cleanup_follows_pagination_and_preserves_other_variables(rtd): + next_url = rtd.api.base + "environmentvariables/?offset=1" + rtd.respond("GET", "environmentvariables/", page({"name": "UNRELATED", "pk": 1}, next=next_url)) + rtd.respond("GET", "environmentvariables/?offset=1", page({"name": guard.CONTEXT_VARIABLE, "pk": 2})) + publisher.clear_context(rtd.api) + assert [(path, data) for method, path, data in rtd.requests if method == "DELETE"] == [ + ("environmentvariables/2/", None), + ] + + +def test_api_keeps_token_out_of_errors_and_rejects_external_urls(rtd): + rtd.respond("GET", "", HTTPError(rtd.api.base, 403, TOKEN, {}, io.BytesIO(TOKEN.encode()))) + with pytest.raises(publisher.APIError) as failure: + rtd.api.request("GET", "") + assert TOKEN not in str(failure.value) + assert failure.value.status == 403 + with pytest.raises(RuntimeError, match="outside"): + rtd.api.request("GET", "https://example.com/") + + +def test_api_reports_invalid_json_response(rtd): + rtd.respond("GET", "", b"Service unavailable") + with pytest.raises(RuntimeError, match="invalid API response"): + rtd.api.request("GET", "") + + +@pytest.mark.parametrize("ref", ["refs/heads/feature", "refs/pull/1/merge", "refs/tags/test"]) +def test_publisher_rejects_refs_outside_release_policy(rtd, ref): + with pytest.raises(RuntimeError, match="Only master"): + publisher.publish(rtd.api, ref, COMMIT) + assert not rtd.requests