diff --git a/.dockerignore b/.dockerignore index c2e4622..ea7dd61 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,7 +1,11 @@ +.git +.venv +cloudbuild + tests/test_init_project.py **/__pycache__/ **/*.pyc **/*.pyo **/*.pyd -.pytest_cache/ \ No newline at end of file +.pytest_cache/ diff --git a/.flake8 b/.flake8 index d311028..794005d 100644 --- a/.flake8 +++ b/.flake8 @@ -2,4 +2,3 @@ max-line-length = 99 exclude = .venv - diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 700b139..7b75c14 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -43,12 +43,13 @@ jobs: cache: 'pip' # cache option make the step fail if you don´t have requirements.txt or pyproject.toml on root. # https://github.com/actions/setup-python/issues/807. - + - name: Install UV run: make uv - name: Create virtual environment - run: uv venv $VENV + # make uv pins .python-version to 3.12; --python makes the venv use the matrix version. + run: uv venv --python ${{ matrix.python-version }} $VENV - name: Install package with test dependencies run: | @@ -62,7 +63,7 @@ jobs: make test - name: Upload coverage reports to Codecov - if: (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop') && matrix.python-version == '3.12' + if: github.ref == 'refs/heads/main' uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} @@ -71,12 +72,12 @@ jobs: name: tests - Python 3.12 on Docker runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python uses: actions/setup-python@v6 - + - name: Build docker image run: make docker-build - + - name: Run tests run: make docker-test diff --git a/.gitignore b/.gitignore index 0838c5a..567258a 100644 --- a/.gitignore +++ b/.gitignore @@ -158,18 +158,8 @@ cython_debug/ #OSX .DS_Store -#Jupyter -.ipynb - #terraform .terraform .terraform.lock.hcl terraform.tfstate terraform.tfstate.backup - -# project stuff -scripts/config.sh -test/ - -# UV -.python-version \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d860096..3951956 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v6.0.0 hooks: - id: check-ast - id: check-added-large-files @@ -15,6 +15,7 @@ repos: args: [--pytest-test-first] - id: mixed-line-ending - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] - repo: local hooks: diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..e4fba21 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d83c281..1e5e7e3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,158 +1,167 @@ -[docker compose]: https://docs.docker.com/compose/install/linux/ -[docker official instructions]: https://docs.docker.com/engine/install/ -[Git Flow]: https://nvie.com/posts/a-successful-git-branching-model/ -[GitHub Flow]: https://githubflow.github.io -[PEP8]: https://peps.python.org/pep-0008/ -[pre-commit]: https://pre-commit.com -[How to Write a Git Commit Message]: https://cbea.ms/git-commit/ -[interactive rebase]: https://git-scm.com/book/en/v2/Git-Tools-Rewriting-History - -[Makefile]: Makefile -[pip-tools]: https://pip-tools.readthedocs.io/en/stable/ -[pyproject.toml]: pyproject.toml -[requirements.txt]: requirements.txt -[requirements-test.txt]: requirements-test.txt - -[GIT-FLOW.md]: docs/contributing/GIT-FLOW.md -[GITHUB-FLOW.md]: docs/contributing/GITHUB-FLOW.md +# How to Contribute -[Preparing the environment]: #preparing-the-environment +[Preparing the development environment]: #preparing-the-development-environment [Updating dependencies]: #updating-dependencies [Git Workflow]: #git-workflow [How to work on a feature branch]: #how-to-work-on-a-feature-branch [How to deploy]: #how-to-deploy - -# How to Contribute +[GitHub Flow]: https://githubflow.github.io/ +[Git Flow]: https://nvie.com/posts/a-successful-git-branching-model/ +[GITHUB-FLOW.md]: docs/contributing/GITHUB-FLOW.md +[GIT-FLOW.md]: docs/contributing/GIT-FLOW.md +[How to Write a Git Commit Message]: https://cbea.ms/git-commit/ +[interactive rebase]: https://git-scm.com/book/en/v2/Git-Tools-Rewriting-History +[pre-commit]: https://pre-commit.com +[PEP8]: https://peps.python.org/pep-0008/ +[docker official instructions]: https://docs.docker.com/engine/install/ +[docker compose]: https://docs.docker.com/compose/install/linux/ +[uv]: https://docs.astral.sh/uv/ +[dependency group]: https://packaging.python.org/en/latest/specifications/dependency-groups/
-These guidelines aim to maintain code quality, -clearly communicate the state of the codebase, +These guidelines apply to GFW's Python repositories created from this template, +pipelines and services alike. +They describe the tooling the template ships (`Makefile`, Docker, uv), +so they live here, next to it, and each repository links to them from its README. +They aim to maintain code quality, clearly communicate the state of the codebase, and standardize the development workflow. -Use your best judgment, and feel free to propose changes -to the guidelines by submitting a pull request. +Use your best judgment, and feel free to propose changes by submitting a pull request.
## Table of Contents -- [Preparing the environment] +- [Preparing the development environment] - [Updating dependencies] - [Git Workflow] - [How to work on a feature branch] - [How to deploy] - -## Preparing the environment +## Preparing the development environment
-Install Docker Engine using the [docker official instructions] (avoid snap packages) -and the [docker compose] plugin. No other system dependencies are required. +This is the common setup shared across repositories created from this template. +It assumes you already cloned the repository, as its README explains, +and that you run every command from the root of the cloned repository. +A given repository's README may add repository-specific system dependencies on top of this +(e.g. GDAL) -- check there first if something doesn't build. -1. First, clone the repository. -```shell -git clone https://github.com/GlobalFishingWatch/python-app-template.git -``` +Install Docker Engine using the [docker official instructions] (avoid snap packages) and +the [docker compose] plugin. + +
+ +1. Make sure you can build the docker image: -2. Make sure you can build the docker image: ```shell make docker-build ``` -3. In order to be able to connect to BigQuery, authenticate and configure the project: +2. In order to be able to connect to Google Cloud (e.g. BigQuery), +authenticate and configure the project: + ```shell make docker-gcp ``` -Install UV for faster installs (otherwise modify Makefile to use regular pip): +> [!NOTE] +> If you'd rather do all development inside a Docker container instead of a local +> virtual environment, run `make docker-shell` to get a shell with everything already +> installed, skip steps 3-6 below, and go directly to step 7 -- `make test` works the +> same way once you're inside that shell as it does in a local venv. + +3. Install [uv] (faster installs than plain pip): + ```shell make uv ``` -4. Create virtual environment and activate it: +4. Create a virtual environment and activate it: + ```shell make venv -./.venv/bin/activate +. .venv/bin/activate ``` -5. Install all dependencies for development and the python package in editable mode: +The activation script must be sourced (`.` or `source`), not executed: +running `./.venv/bin/activate` would only affect a subshell. + +5. Install all dependencies and the package itself in editable mode: + ```shell make install ``` -6. (Optional) Install [pre-commit] hooks: +6. (Optional) Install the [pre-commit] hooks: + ```shell make hooks ``` -This step is strongly recommended to maintain code quality and prevent technical debt, + +This is strongly recommended to maintain code quality and prevent technical debt, particularly regarding documentation, [PEP8] compliance, spelling, and type-hinting. -If this feels too rigid for your current workflow, -at a minimum, make regular use of the provided [Makefile] commands: -`format`, `lint`, `codespell`, and `typecheck`. +If this feels too rigid for your current workflow, at a minimum, +make regular use of the `Makefile`'s `format`, `lint`, `codespell`, and `typecheck` targets. +[PEP8] checks are enforced in the GitHub CI. -[PEP8] checks will be enforced in the GitHub CI. +7. Make sure you can run the unit tests: -7. Make sure you can run unit tests: ```shell make test ``` - - -> [!NOTE] -> Alternatively, - you can handle all the development inside a docker container - without installing dependencies in a virtual environment. - Use `make docker-shell` to enter a docker container. - ## Updating dependencies
-The [requirements.txt] file contains all transitive dependencies pinned to specific versions. -It is automatically generated using [pip-tools], -based on the dependencies specified in [pyproject.toml]. -This process ensures reproducibility, -allowing the application to run consistently across different environments. +`requirements.txt` contains all transitive dependencies pinned to specific versions. It +is automatically generated with `uv pip compile`, based on the dependencies specified in +`pyproject.toml`. This process ensures reproducibility, allowing the application to run +consistently across different environments. -Use [pyproject.toml] to define high-level dependencies with flexible version constraints -(e.g., ~=1.2, >=1.0, <2.0, ...). +`requirements-test.txt` does the same for the test dependencies, declared in the `test` +[dependency group] of `pyproject.toml`. It is compiled against the pins in `requirements.txt`, +so the test image can install just the test dependencies on top of the production image +and test the package exactly as it is deployed. -To re-compile dependencies, just run -```shell -make reqs -``` +Use `pyproject.toml` to define high-level dependencies with flexible version constraints +(e.g., `~=1.2`, `>=1.0`, `<2.0`, ...). +`make reqs-upgrade` upgrades within those constraints; +a new major version (e.g. `~=8.3` to `~=9.0`) is a deliberate change in `pyproject.toml`. + +Every repository's `Makefile` is self-documenting -- run `make help` (or bare `make`) to +see its exact current targets and descriptions. The typical names are: -If you want to upgrade all dependencies to latest compatible versions, just run: ```shell -make reqs-upgrade +make reqs # re-compile requirements.txt and requirements-test.txt +make reqs-upgrade # upgrade all dependencies to their latest compatible versions ``` +
> [!IMPORTANT] -> Do not modify [requirements.txt] manually. +> Do not modify `requirements.txt` or `requirements-test.txt` manually. > [!NOTE] -> Remember that if you change the [requirements.txt], -you need to rebuild the docker image (`make docker-build`) in order to use it locally. +> After changing `requirements.txt`, rebuild the docker image (`make docker-build`) +> to use it locally. +> If the project uses Apache Beam, keep the Beam SDK image in the `Dockerfile` +> at the same version as the `apache-beam` pin in `requirements.txt`. ## Git Workflow
-There are two main Git workflows commonly used in software development: -[Git Flow] and [GitHub Flow]. -We encourage using **GitHub Flow** whenever possible, -as it is a lightweight workflow that supports rapid iteration, -continuous delivery, and simpler branching. -It works especially well for teams that deploy frequently -and want to keep their main branch always production-ready. -However, -if your project involves a long release cycle and a long release process, -or you need an unstable shared branch like develop for ongoing integration, -then **Git Flow** may be more appropriate. -You can read a summary of both workflows in [GITHUB-FLOW.md] and [GIT-FLOW.md]. +There are two main Git workflows commonly used in software development: [Git Flow] and +[GitHub Flow]. We encourage using **GitHub Flow** whenever possible, as it's a +lightweight workflow that supports rapid iteration, continuous delivery, and simpler +branching. It works especially well for teams that deploy frequently and want to keep +`main` always production-ready. However, if a project involves a long release cycle and +a long release process, or needs an unstable shared branch like `develop` for ongoing +integration, then **Git Flow** may be more appropriate. You can read a summary of both +workflows in [GITHUB-FLOW.md] and [GIT-FLOW.md].
@@ -160,39 +169,27 @@ You can read a summary of both workflows in [GITHUB-FLOW.md] and [GIT-FLOW.md].
-To work on a new feature or fix, -create a branch from `main` or `develop`, -depending on the workflow your team follows. - -Try to follow these guidelines: - -- Branch names should be descriptive and concise, - all lowercase and with words separated by hyphens "-". - Optionally, feature branch names can be prefixed with **JIRA** ticket. - For example, you can use something like `PIPELINE-2020-name-of-the-branch`. +To work on a new feature or fix, create a branch from `main` (or `develop`, depending +on the workflow the repository follows). +- Branch names should be descriptive and concise, all lowercase and with words separated + by hyphens. Optionally, prefix with the **JIRA** ticket, e.g. + `PIPELINE-2020-name-of-the-branch`. - Write clear commit messages. See [How to Write a Git Commit Message]. - -- Maintain a clean commit history in your feature branch. - Use interactive rebase (`git rebase -i`) to squash, reorder, or edit commits.[^1] - -- If you are not using [pre-commit] hooks, - use the provided [Makefile] commands (`format`, `lint`, `codespell`, `typecheck`) - as much as possible to maintain code quality. - -- Add unit tests for each piece of code: - * Avoid connecting to external services during unit tests. Use mocks as needed. - * Ensure unit tests run as fast as possible. - -- When submitting a Pull Request (**PR**), ensure it meets the following criteria: - * It targets the correct base branch (depending on the chosen [Git Workflow]). - * It is focused and limited in scope to simplify the review process. - * Its description includes a link to the related **JIRA** ticket - to support integration between the issue and the **PR**. - -[^1]: When performing a rebase, it's acceptable to `push --force` to your own feature branch, - but **never** on shared branches. - Protect `main` and/or `develop` from force pushes via GitHub settings. +- Maintain a clean commit history in your feature branch. Use [interactive rebase] + (`git rebase -i`) to squash, reorder, or edit commits.[^1] +- If you're not using [pre-commit] hooks, run the `Makefile`'s quality-check targets + (`format`, `lint`, `codespell`, `typecheck`) as much as possible before pushing. +- Add unit tests for each piece of code: avoid connecting to external services during + tests (mock as needed), and keep tests fast. +- When opening a pull request: + - Target the correct base branch, depending on the repository's [Git Workflow]. + - Keep it focused and limited in scope to simplify review. + - Link the related **JIRA** ticket in the description. + +[^1]: When performing a rebase, it's acceptable to `push --force` to your own feature + branch, but **never** on shared branches. Protect `main` and/or `develop` from + force pushes via GitHub settings.
@@ -200,9 +197,12 @@ Try to follow these guidelines:
-A Google Cloud build that publishes a Docker image is triggered in the following cases: -- When a commit is merged into `main` or `develop`. -- When a new tag is created. +A Google Cloud Build that publishes a Docker image is triggered in either of these +cases: + +- A commit is merged into `main` (or `develop`, for repositories using Git Flow). +- A new tag is created. +The triggers are defined with Terraform in the repository's `cloudbuild/` directory.
diff --git a/Dockerfile b/Dockerfile index 0d36072..34cfa8c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ COPY --from=ghcr.io/astral-sh/uv:0.10.9 /uv /usr/local/bin/uv ENV UV_COMPILE_BYTECODE=1 # Install dependencies BEFORE copying source so an edit under src/ -# doesn't invalidate the cache of the (expensive) requirements-install layer. +# doesn't invalidate the cache of the (expensive) requirements-install layer. COPY pyproject.toml requirements.txt README.md MANIFEST.in ./ RUN uv pip install --system --upgrade pip && \ uv pip install --system build && \ @@ -31,6 +31,7 @@ ENV PYTHONUNBUFFERED=1 COPY --from=builder /install /usr/local # APACHE BEAM INTEGRATION (Uncomment if needed) +# IMPORTANT: this version must match the apache-beam pin in requirements.txt. # COPY --from=apache/beam_python3.12_sdk:2.71.0 /opt/apache/beam /opt/apache/beam # ENTRYPOINT ["/opt/apache/beam/boot"] @@ -63,4 +64,4 @@ COPY ./tests ./tests # Suppress all warnings during tests # To see/address warnings, run tests in your development environment. -ENV PYTHONWARNINGS=ignore \ No newline at end of file +ENV PYTHONWARNINGS=ignore diff --git a/Makefile b/Makefile index 24ceca1..de7bf2a 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ VENV_NAME:=.venv REQS_PROD:=requirements.txt +REQS_TEST:=requirements-test.txt SETUP_FILE:=pyproject.toml SOURCES = src @@ -13,6 +14,9 @@ DOCKER_TEST_SERVICE:=test GCP_PROJECT:=world-fishing-827 GCP_DOCKER_VOLUME:=gcp +# Run as the host user so files written into the mounted repo aren't owned by root. +DOCKER_AS_USER:=--user $(shell id -u):$(shell id -g) -e UV_CACHE_DIR=/tmp/uv-cache + PYTHON_VERSION:=3.12 UV_VERSION := 0.10.9 @@ -48,22 +52,24 @@ docker-test: docker-shell: docker-volume docker compose run --rm -it ${DOCKER_DEV_SERVICE} -.PHONY: reqs ## Compiles requirements.txt with pip-tools. +.PHONY: reqs ## Compiles requirements.txt and requirements-test.txt with uv. reqs: - docker compose run --rm ${DOCKER_DEV_NO_GCP_SERVICE} -c \ - '${PIP_COMPILE} -o ${REQS_PROD} ${SETUP_FILE} -v' + docker compose run --rm ${DOCKER_AS_USER} ${DOCKER_DEV_NO_GCP_SERVICE} -c \ + '${PIP_COMPILE} -o ${REQS_PROD} ${SETUP_FILE} -v && \ + ${PIP_COMPILE} --group test -c ${REQS_PROD} -o ${REQS_TEST} ${SETUP_FILE} -v' -.PHONY: reqs-upgrade ## Upgrades requirements.txt with pip-tools. +.PHONY: reqs-upgrade ## Upgrades requirements.txt and requirements-test.txt with uv. reqs-upgrade: - docker compose run --rm ${DOCKER_DEV_NO_GCP_SERVICE} -c \ - '${PIP_COMPILE} -o ${REQS_PROD} ${SETUP_FILE} -U -v' + docker compose run --rm ${DOCKER_AS_USER} ${DOCKER_DEV_NO_GCP_SERVICE} -c \ + '${PIP_COMPILE} -o ${REQS_PROD} ${SETUP_FILE} -U -v && \ + ${PIP_COMPILE} --group test -c ${REQS_PROD} -o ${REQS_TEST} ${SETUP_FILE} -U -v' # --------------------- # VIRTUAL ENVIRONMENT # --------------------- .PHONY: uv ## Installs UV -uv: +uv: curl -LsSf https://astral.sh/uv/install.sh | UV_VERSION=$(UV_VERSION) sh uv python pin ${PYTHON_VERSION} @@ -95,7 +101,6 @@ test: .PHONY: hooks ## Install and pre-commit hooks. hooks: python -m pre_commit install --install-hooks - python -m pre_commit install --hook-type commit-msg .PHONY: format ## Auto-format python source files according with PEP8. format: @@ -111,7 +116,7 @@ lint: .PHONY: codespell ## Use Codespell to do spell checking. codespell: - python -m codespell + python -m codespell_lib .PHONY: typecheck ## Perform type-checking. typecheck: diff --git a/README.md b/README.md index d81548f..fd51ac8 100644 --- a/README.md +++ b/README.md @@ -19,12 +19,12 @@ A template for Python applications. **Features**: * :white_check_mark: Standard Python project structure & packaging. -* :white_check_mark: Dependency management with [pip-tools]. +* :white_check_mark: Dependency management with [uv]. * :white_check_mark: Tools for quality checks: documentation, [PEP8], typehints, codespell. * :white_check_mark: **[Optional]** pre-commit hooks to enforce automatic quality checks. * :white_check_mark: Dockerization with focus in image size optimization. * :white_check_mark: Continuous Integration (CI) workflows (GitHub Actions). -* :white_check_mark: Continuous Deployment (CI) workflows (Google Cloud Build). +* :white_check_mark: Continuous Deployment (CD) workflows (Google Cloud Build). * :white_check_mark: Makefile with shortcuts to increase development speed. * :white_check_mark: README badges with project information. * :white_check_mark: Development workflow documentation. @@ -40,20 +40,19 @@ A template for Python applications. [Markdown alerts]: https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax#alerts [mermaid diagram]: https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/creating-diagrams [PEP8]: https://peps.python.org/pep-0008/ -[pip-tools]: https://pip-tools.readthedocs.io/en/stable/ +[uv]: https://docs.astral.sh/uv/ [pre-commit]: https://pre-commit.com [pytest]: https://docs.pytest.org/en/stable/ [used by pytest-cov]: https://pytest-cov.readthedocs.io/en/latest/config.html -[cli.py]: src/python_app_template/cli.py -[docs/contributing]: docs/contributing -[examples]: examples/ - +[cli/main.py]: src/python_app_template/cli/main.py + [.github/]: .github/ +[cloudbuild/]: cloudbuild/ [docs/]: docs/ [notebooks/]: notebooks/ -[src/python_app_template/]: src/python_app_template +[src/python_app_template/]: src/python_app_template [src/python_app_template/assets/]: src/python_app_template/assets/ [tests/]: tests/ @@ -62,18 +61,17 @@ A template for Python applications. [.gitignore]: .gitignore [.pre-commit-config.yaml]: .pre-commit-config.yaml [activate-venv.sh]: activate-venv.sh -[cloudbuild.yaml]: cloudbuild.yaml [codecov.yml]: codecov.yml [CONTRIBUTING.md]: CONTRIBUTING.md +[contributing guide]: https://github.com/GlobalFishingWatch/python-app-template/blob/main/CONTRIBUTING.md [docker-compose.yml]: docker-compose.yml [Dockerfile]: Dockerfile -[Git Workflow]: CONTRIBUTING.md#git-workflow +[Git Workflow]: https://github.com/GlobalFishingWatch/python-app-template/blob/main/CONTRIBUTING.md#git-workflow [init_project.py]: init_project.py [LICENSE]: LICENSE [Makefile]: Makefile [MANIFEST.in]: MANIFEST.in [pyproject.toml]: pyproject.toml -[pytest.ini]: pytest.ini [README.md]: README.md [requirements.txt]: requirements.txt [requirements-test.txt]: requirements-test.txt @@ -96,7 +94,7 @@ Goals of this template include: * Minimizing the effort required to transition prototypes to production. * Establishing consistent development standards across projects. -## Usage +## Usage _Write the relevant documentation on how to use this application here. Ideally, organize it into sections_. @@ -124,7 +122,8 @@ and incrementally builds upon the requirements of the previous stages. 1. 💡 **Proof of Concept** - Update the project's name with `python init_project.py my-project-name`. - Declare required dependencies in [pyproject.toml]. - Use the `dev` extra for development-only dependencies. + Use the `dev` extra for development-only dependencies, + and the `test` dependency group for test dependencies. - Store data files (e.g., `.txt`, `.json`, `.csv`) in the `src/my_project_name/assets/` directory. - Add the following sections to the `README.md`: - **Introduction** @@ -133,7 +132,7 @@ and incrementally builds upon the requirements of the previous stages. 2. 🛠️ **Prototype** - Set up [branch protection rules] for `main` branch (and `develop` if exists). - ☑️ Restrict deletions. - - ☑️ Require a pull request before merging. + - ☑️ Require a pull request before merging. - ☑️ Require approvals. - ☑️ Require conversation resolution before merging. - ☑️ Require status checks to pass. Add GitHub actions to checks. @@ -141,7 +140,8 @@ and incrementally builds upon the requirements of the previous stages. - ☑️ Block force pushes. - ☑️ Allowed merge methods: only Merge. - Select a [Git Workflow] to maintain consistent branching and collaboration practices. - - Set up Google Cloud Build triggers to automatically publish the Docker image upon merges. + - Create the Google Cloud Build triggers that publish the Docker image upon merges and tags: + run `terraform init` and `terraform apply` in [cloudbuild/]. - Re-compile `requirements.txt` for Docker using: ``` make reqs @@ -150,7 +150,7 @@ and incrementally builds upon the requirements of the previous stages. - **Output Schema** 3. 🚀 **Production** - - Install and configure pre-commit hooks. See [CONTRIBUTING.md]. + - Install and configure pre-commit hooks. See the [contributing guide]. - Add unit tests to ensure code reliability. - Write thorough documentation: - A complete `README.md`. @@ -171,6 +171,7 @@ This is a brief summary of all the relevant directories of the repository. | Directory | Description | | ------------------------------- | ------------------------------------------------------------------------------- | |[.github/] | Configuration for GitHub actions. | +|[cloudbuild/] | Terraform config for the Cloud Build triggers. | |[docs/] | Markdown files with detailed documentation. | |[notebooks/] | All jupyter notebooks go here. | |[src/python_app_template/] | All source code go here. | @@ -192,9 +193,8 @@ This is a brief summary of all the relevant files of the repository. |[.gitignore] | List of files and directories to be ignored by git. | |[.pre-commit-config.yaml] | Configuration to automate software quality checks. | |[activate-venv.sh] | Simple shortcut to enter virtual environment. | -|[cloudbuild.yaml] | Configuration to build and publish docker images in Google Cloud. | |[codecov.yml] | Configuration for [codecov] GitHub integration. | -|[CONTRIBUTING.md] | A guide for contributing to the project. | +|[CONTRIBUTING.md] | Contributing guide shared by all projects created from this template. | |[docker-compose.yml] | Configuration for [docker compose]. | |[Dockerfile] | Instructions for building the Docker image. | |[init_project.py] | Automates changing the project name across relevant files and directories. | @@ -203,7 +203,7 @@ This is a brief summary of all the relevant files of the repository. |[MANIFEST.in] | Set of patterns to include or exclude files from installed package. | |[pyproject.toml] | Modern Python packaging configuration file. | |[requirements.txt] | Full set of compiled production dependencies (pinned to specific versions). | -|[requirements-test.txt] | High level test dependencies needed to test the installed package. | +|[requirements-test.txt] | Compiled test dependencies, installed on top of the production image to test it.| |[README.md] | This file. | |[setup.py] | Legacy Python packaging config file, kept for compatibility with [Apache Beam]. | @@ -220,7 +220,7 @@ _Write instructions on how to use the CLI of the application here._ _**Optional**_. _Provide an example of an input configuration file._ -## Output schema. +## Output schema _**Optional**_. _Discuss any relevant details about the schema. @@ -233,7 +233,16 @@ _Explain the data persistence pattern used in this application._ ## How to Contribute -Please read the guidelines in [CONTRIBUTING.md]. +First, clone the repository: + +```shell +git clone https://github.com/GlobalFishingWatch/python-app-template.git +cd python-app-template +``` + +Then, from the cloned repository, follow the [contributing guide] +(shared by all projects created from this template) +to prepare the development environment and learn the git workflow. ## Implementation details @@ -250,7 +259,7 @@ Example: | Module | Description | | --- | --- | -| [cli.py] | Defines the application CLI. | +| [cli/main.py] | Defines the application CLI. | @@ -268,4 +277,4 @@ Here we provide some examples._ [1] Welch H., Clavelle T., White T. D., Cimino M. A., Van Osdel J., Hochberg T., et al. (2022). Hot spots of unseen fishing vessels. Sci. Adv. 8 (44), eabq2109. doi: 10.1126/sciadv.abq2109 -[2] J. H. Ford, B. Bergseth, C. Wilcox, Chasing the fish oil—Do bunker vessels hold the key to fisheries crime networks? Front. Mar. Sci. 5, 267 (2018). \ No newline at end of file +[2] J. H. Ford, B. Bergseth, C. Wilcox, Chasing the fish oil—Do bunker vessels hold the key to fisheries crime networks? Front. Mar. Sci. 5, 267 (2018). diff --git a/cloudbuild.yaml b/cloudbuild.yaml deleted file mode 100644 index 63c33e5..0000000 --- a/cloudbuild.yaml +++ /dev/null @@ -1,22 +0,0 @@ -steps: - - name: gcr.io/cloud-builders/docker - id: build - args: - - build - - '-t' - - '${_IMAGE_NAME}:${TAG_NAME}' - - '-t' - - '${_IMAGE_NAME}:latest' - - '-f' - - Dockerfile - - '--target' - - prod - - . -images: - - '${_IMAGE_NAME}:${TAG_NAME}' - - '${_IMAGE_NAME}:latest' -timeout: 600s -substitutions: - _IMAGE_NAME: 'gcr.io/${PROJECT_ID}/github.com/globalfishingwatch/python-app-template' -options: - dynamic_substitutions: true diff --git a/cloudbuild/backend.tf b/cloudbuild/backend.tf new file mode 100644 index 0000000..c6cd49a --- /dev/null +++ b/cloudbuild/backend.tf @@ -0,0 +1,6 @@ +terraform { + backend "gcs" { + bucket = "gfw-int-infrastructure-tfstate-us-central1" + prefix = "cloudbuild-python-app-template" + } +} diff --git a/cloudbuild/main.tf b/cloudbuild/main.tf new file mode 100644 index 0000000..f92f421 --- /dev/null +++ b/cloudbuild/main.tf @@ -0,0 +1,15 @@ +module "trigger_push_to_tag" { + source = "git::https://github.com/GlobalFishingWatch/gfw-terraform-modules.git//modules/cloudbuild-trigger?ref=v0.6.7" + registry_artifact = "core" + repo_name = "python-app-template" + tag = ".*" + trigger_description = "Builds and publishes a Docker image on every tag push." +} + +module "trigger_push_to_main" { + source = "git::https://github.com/GlobalFishingWatch/gfw-terraform-modules.git//modules/cloudbuild-trigger?ref=v0.6.7" + registry_artifact = "core" + repo_name = "python-app-template" + branch = "main" + trigger_description = "Builds and publishes a Docker image on every push to the main branch." +} diff --git a/codecov.yml b/codecov.yml index 334b6cd..54aa186 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,4 +1,4 @@ coverage: # Avoid failing CI jobs because of coverage. status: patch: off - project: off \ No newline at end of file + project: off diff --git a/docs/contributing/GIT-FLOW.md b/docs/contributing/GIT-FLOW.md index 6179617..51ba7ce 100644 --- a/docs/contributing/GIT-FLOW.md +++ b/docs/contributing/GIT-FLOW.md @@ -10,7 +10,7 @@ In the following, **X**, **Y** and **Z** refer to **MAJOR**, **MINOR** and **PAT These are the 5 types of branches used in this strategy: | Name | Type | Purpose | -|-----------------|-----------|----------------------------------------------------------------------------------| +|-----------------|-----------|------------------------------------------------------------------------------------| | `main` | Permanent | Represents the production-ready state; all releases originate here. | | `develop` | Permanent | The integration branch for ongoing development; features are merged here. | | `feature/*` | Temporary | Branches for developing new features, branched off from `develop`. | diff --git a/docs/contributing/GITHUB-FLOW.md b/docs/contributing/GITHUB-FLOW.md index 111b855..d22a15a 100644 --- a/docs/contributing/GITHUB-FLOW.md +++ b/docs/contributing/GITHUB-FLOW.md @@ -60,8 +60,11 @@ This flow emphasizes **simplicity, speed, and collaboration**, especially for te git merge branch_name --no-ff ``` +- Maintain a clean commit history in your feature branch. + Use interactive rebase (`git rebase -i`) to squash, reorder, or edit commits.[^1] + [^1]: When performing a rebase, it's acceptable to `push --force` to your own feature branch, but **never** on shared branches. - Protect `main` and/or `develop` from force pushes via GitHub settings. + Protect `main` and/or `develop` from force pushes via GitHub settings. diff --git a/init_project.py b/init_project.py index 47f8112..c5f0e11 100644 --- a/init_project.py +++ b/init_project.py @@ -1,12 +1,19 @@ """Renames a Python project by replacing the template name in files and directories.""" import sys +import shutil import pathlib OLD_NAME = "python-app-template" +# The contributing guide is shared by all projects created from this template: +# new projects link to it instead of keeping a copy. +CONTRIBUTING_GUIDE_URL = ( + f"https://github.com/GlobalFishingWatch/{OLD_NAME}/blob/main/CONTRIBUTING.md" +) + FILES_TO_UPDATE = ( - "cloudbuild.yaml", - "CONTRIBUTING.md", + "cloudbuild/backend.tf", + "cloudbuild/main.tf", "docker-compose.yml", "Makefile", "pyproject.toml", @@ -17,6 +24,11 @@ "tests/cli/test_main.py", ) +PATHS_TO_REMOVE = ( + "CONTRIBUTING.md", + "docs/contributing", +) + VERSION_FILE_TPL = "{src_dir}/{old_name}/version.py" SOURCE_DIR = pathlib.Path("src") @@ -48,11 +60,25 @@ def __str__(self): def _replace_in_file(filepath: pathlib.Path, old_new_pairs: list[tuple[str, str]]) -> None: - content = filepath.read_text(encoding="utf-8") + # Links to the shared contributing guide must keep pointing to the template. + parts = filepath.read_text(encoding="utf-8").split(CONTRIBUTING_GUIDE_URL) for old, new in old_new_pairs: - content = content.replace(old, new) + parts = [part.replace(old, new) for part in parts] - filepath.write_text(content, encoding="utf-8") + filepath.write_text(CONTRIBUTING_GUIDE_URL.join(parts), encoding="utf-8") + + +def _remove_path(path: pathlib.Path) -> None: + if not path.exists(): + print(f"⏭️ Skipped (not found): {path}") + return + + if path.is_dir(): + shutil.rmtree(path) + else: + path.unlink() + + print(f"🗑️ Removed: {path}") def _rename_package_dir(src_dir: pathlib.Path, old_name: str, new_name: str) -> None: @@ -100,11 +126,18 @@ def update_project_name( print(f"🎉 Project renamed to: {new_name.with_dashes}.") +def remove_template_only_paths(paths_to_remove: tuple[str, ...] = PATHS_TO_REMOVE) -> None: + """Removes files that only belong to the template, like the shared contributing guide.""" + for path in paths_to_remove: + _remove_path(pathlib.Path(path)) + + def main( args: list[str] = sys.argv, old_name: str = OLD_NAME, files_to_update: tuple[str, ...] = FILES_TO_UPDATE, - src_dir: pathlib.Path = SOURCE_DIR + src_dir: pathlib.Path = SOURCE_DIR, + paths_to_remove: tuple[str, ...] = PATHS_TO_REMOVE, ): """Entry point for the script. @@ -129,6 +162,7 @@ def main( files_to_update=tuple(files_to_update), src_dir=src_dir ) + remove_template_only_paths(paths_to_remove) if __name__ == "__main__": diff --git a/pyproject.toml b/pyproject.toml index ec28501..9d3ec3b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,23 +24,16 @@ maintainers = [ ] classifiers = [ "Operating System :: OS Independent", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", - -] -requires-python = ">= 3.9" -dependencies = [ - "pyyaml~=6.0", - "rich~=14.0", - "rich-argparse~=1.6", ] +requires-python = ">= 3.12" +dependencies = [] [project.urls] Homepage = "https://github.com/GlobalFishingWatch/python-app-template" Documentation = "https://globalfishingwatch.github.io/python-app-template/" -Changelog = "https://github.com/GlobalFishingWatch/python-app-template/blob/main/CHANGELOG.md" +Changelog = "https://github.com/GlobalFishingWatch/python-app-template/releases" Repository = "https://github.com/GlobalFishingWatch/python-app-template" Issues = "https://github.com/GlobalFishingWatch/python-app-template/issues" @@ -57,13 +50,11 @@ lint = [ "ruff~=0.11", # Linter and code analysis tool. "codespell[toml]~=2.4", # Spell checker for code. "flake8~=7.0", # Simple PEP8 checker. - "types-PyYAML", # MyPy stubs for pyyaml. ] # Development workflow and tools dev = [ "pre-commit~=4.2", # Framework for managing pre-commit hooks. - "pip-tools>=7.4,<8.0", # Compatible with pip 24+, fixes use_pep517 AttributeError. "pip-audit~=2.8", # Audit for finding vulnerabilities in dependencies. ] @@ -74,6 +65,15 @@ build = [ "twine~=6.1", # For uploading Python packages to PyPI. ] +# Test dependencies (PEP 735). Not published with the package: the test Docker image installs +# them, compiled into requirements-test.txt, on top of the production image. +[dependency-groups] +test = [ + "pytest~=9.0", # Core testing framework. + "pytest-cov~=7.0", # Coverage plugin for pytest. + "pytest-mock~=3.14", # Mocking plugin for pytest. +] + [tool.ruff] fix = true line-length = 99 @@ -144,7 +144,6 @@ addopts = "-v --cov=python_app_template --cov=init_project --cov-report=term-mis [tool.coverage.run] branch = true -parallel = true context = "${CONTEXT}" [tool.coverage.report] @@ -166,4 +165,4 @@ exclude_lines = [ ] [tool.codespell] -skip = '.git,env*,venv*,.venv*, build*,tmp*' \ No newline at end of file +skip = '.git,env*,venv*,.venv*, build*,tmp*' diff --git a/requirements-test.txt b/requirements-test.txt index b0c47c5..13bc6f2 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,7 +1,23 @@ -# We want to be able to install only test dependencies, for example when testing an already installed package. -# Currently, is not possible to install only dependencies from pyproject.toml. -# For that, we use this file to specify the test dependencies required. - -pytest~=8.3 # Core testing framework. -pytest-cov~=6.0 # Coverage plugin for pytest. -pytest-mock~=3.14 # Mocking plugin for pytest. +# This file was autogenerated by uv via the following command: +# uv pip compile --group test -c requirements.txt -o requirements-test.txt pyproject.toml +coverage==7.16.2 + # via pytest-cov +iniconfig==2.3.1 + # via pytest +packaging==26.3 + # via pytest +pluggy==1.7.0 + # via + # pytest + # pytest-cov +pygments==2.21.0 + # via pytest +pytest==9.1.1 + # via + # python-app-template (pyproject.toml:test) + # pytest-cov + # pytest-mock +pytest-cov==7.1.0 + # via python-app-template (pyproject.toml:test) +pytest-mock==3.16.0 + # via python-app-template (pyproject.toml:test) diff --git a/requirements.txt b/requirements.txt index b403fac..0f0c262 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,16 +1,2 @@ # This file was autogenerated by uv via the following command: # uv pip compile -o requirements.txt pyproject.toml -markdown-it-py==4.0.0 - # via rich -mdurl==0.1.2 - # via markdown-it-py -pygments==2.19.2 - # via rich -pyyaml==6.0.3 - # via python-app-template (pyproject.toml) -rich==14.2.0 - # via - # python-app-template (pyproject.toml) - # rich-argparse -rich-argparse==1.7.2 - # via python-app-template (pyproject.toml) diff --git a/src/python_app_template/assets/data.json b/src/python_app_template/assets/data.json index dcb9714..b83a0a7 100644 --- a/src/python_app_template/assets/data.json +++ b/src/python_app_template/assets/data.json @@ -1,3 +1,3 @@ { "value": 1234 -} \ No newline at end of file +} diff --git a/tests/test_init_project.py b/tests/test_init_project.py index ebb578b..3b6de33 100644 --- a/tests/test_init_project.py +++ b/tests/test_init_project.py @@ -27,7 +27,17 @@ def test_main_renames_project(tmp_path): # Create a single additional file to update readme = tmp_path / "README.md" - readme.write_text("Welcome to python-app-template!\n") + readme.write_text( + "Welcome to python-app-template!\n" + f"See the [contributing guide]({init_project.CONTRIBUTING_GUIDE_URL}).\n" + ) + + # Create the template-only files + contributing = tmp_path / "CONTRIBUTING.md" + contributing.write_text("# How to Contribute\n") + contributing_docs = tmp_path / "docs" / "contributing" + contributing_docs.mkdir(parents=True) + (contributing_docs / "GITHUB-FLOW.md").write_text("# GitHub Flow\n") # Run main() args = ["init_project.py", new_name] @@ -36,14 +46,20 @@ def test_main_renames_project(tmp_path): args=args, old_name=old_name, files_to_update=files_to_update, - src_dir=src_dir + src_dir=src_dir, + paths_to_remove=(str(contributing), str(contributing_docs)), ) # Check that file was updated updated_readme = readme.read_text() print(updated_readme) assert "my-project" in updated_readme - assert "python-app-template" not in updated_readme + assert "Welcome to my-project!" in updated_readme + assert init_project.CONTRIBUTING_GUIDE_URL in updated_readme + + # Check that template-only files were removed + assert not contributing.exists() + assert not contributing_docs.exists() # Check that version file was updated new_pkg_dir = src_dir / new_dir_name @@ -134,3 +150,12 @@ def test_main_usage_message_on_bad_args(capsys): assert expected_usage in captured.out assert exc_info.value.code == 1 + + +def test_remove_template_only_paths_skips_missing_path(tmp_path, capsys): + missing_path = tmp_path / "missing_dir" + + init_project.remove_template_only_paths(paths_to_remove=(str(missing_path),)) + + captured = capsys.readouterr() + assert f"⏭️ Skipped (not found): {missing_path}" in captured.out