From ff4812f218f135158757db8e348d14adc6a0d98b Mon Sep 17 00:00:00 2001 From: AntoninKns Date: Wed, 11 Nov 2026 20:46:55 +0300 Subject: [PATCH 1/3] Update quick example tutorial --- documentation/index.md | 2 +- documentation/mkdocs.yaml | 2 +- documentation/tutorials/getting-started.md | 83 ++++++---------------- 3 files changed, 22 insertions(+), 65 deletions(-) diff --git a/documentation/index.md b/documentation/index.md index 101ef93..716ce5b 100755 --- a/documentation/index.md +++ b/documentation/index.md @@ -10,7 +10,7 @@ upload your datasets to HDX. ## Tutorials -- [A Quick Example](tutorials/getting-started.md) +- [Read a Dataset](tutorials/getting-started.md) ## How-to Guides diff --git a/documentation/mkdocs.yaml b/documentation/mkdocs.yaml index 465a284..60aee1f 100644 --- a/documentation/mkdocs.yaml +++ b/documentation/mkdocs.yaml @@ -12,7 +12,7 @@ plugins: nav: - Home: index.md - Tutorials: - - A Quick Example: tutorials/getting-started.md + - Read a Dataset: tutorials/getting-started.md - How-to Guides: - Set up: - Obtain your API Key: how-to/api-key.md diff --git a/documentation/tutorials/getting-started.md b/documentation/tutorials/getting-started.md index ec4d69e..ebffbfa 100644 --- a/documentation/tutorials/getting-started.md +++ b/documentation/tutorials/getting-started.md @@ -1,20 +1,12 @@ -# A Quick Example +# Read a Dataset Let's start with a simple example that also ensures that the library is working -properly. In this tutorial, we use virtualenv, a sandbox, so that your Python install is -not modified. +properly. In this tutorial, we read data from HDX, which requires no account and no API +key. We use virtualenv, a sandbox, so that your Python install is not modified. -1. If you just want to read data from HDX, then an API key is not necessary. However, -if you want to write data to HDX, then you need to register on the website to obtain an -API key. Please see [Obtain your API Key](../how-to/api-key.md) about where to find it on the website. Once you have it, then -put it into a file in your home directory: - - cd ~ - echo "hdx_key: \"xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx\"" > .hdx_configuration.yaml - -2. If you are using the Docker image, you can jump to step 6, otherwise install +1. If you are using the Docker image, you can jump to step 5, otherwise install virtualenv if not installed: pip install virtualenv @@ -24,7 +16,7 @@ virtualenv if not installed: sudo apt-get install virtualenv -3. Create a Python 3 virtualenv and activate it: +2. Create a Python 3 virtualenv and activate it: On Windows (assuming the Python 3 executable is in your path): @@ -36,90 +28,51 @@ virtualenv if not installed: virtualenv -p python3 test source test/bin/activate -4. Install the HDX Python library: +3. Install the HDX Python library: pip install hdx-python-api -5. If you get errors, it is probably the +4. If you get errors, it is probably the [dependencies of the cryptography package](../how-to/install.md) -6. Launch python: +5. Launch python: python -7. Import required classes: +6. Import required classes: from hdx.utilities.easy_logging import setup_logging from hdx.api.configuration import Configuration from hdx.data.dataset import Dataset -8. Setup logging +7. Setup logging setup_logging() -9. Use configuration defaults. - - If you only want to read data, then connect to the production HDX - server, making sure that you replace MyOrg_MyProject with something that - describes your organisation and project: +8. Connect to the production HDX server in read-only mode, making sure that you + replace MyOrg_MyProject with something that describes your organisation and project: Configuration.create(hdx_site="prod", user_agent="MyOrg_MyProject", hdx_read_only=True) - If you want to write data, then for experimentation, do not use the - production HDX server. Instead you can use one of the test servers. - Assuming you have an API key stored in a file **.hdxkey** in the - current user's home directory: - - Configuration.create(hdx_site="stage", user_agent="MyOrg_MyProject") - -10. Read this dataset +9. Read this dataset [Novel Coronavirus (COVID-19) Cases Data](https://data.humdata.org/dataset/novel-coronavirus-2019-ncov-cases) from HDX and view the date of the dataset: dataset = Dataset.read_from_hdx("novel-coronavirus-2019-ncov-cases") print(dataset.get_time_period()) -11. You can search for datasets on HDX and get their resources: +10. You can search for datasets on HDX and get their resources: datasets = Dataset.search_in_hdx("thailand subnational boundaries", rows=10) print(datasets) resources = Dataset.get_all_resources(datasets) print(resources) -12. You can download a resource in the dataset: +11. You can download a resource in the dataset: url, path = resources[0].download() print("Resource URL %s downloaded to %s" % (url, path)) -13. If you have an API key, you can write to HDX. You can try it out on a test - server. With a dataset to which you have permissions, change the dataset date: - - dataset = Dataset.read_from_hdx("ID OR NAME OF DATASET") - print(dataset.get_time_period()) # record this - dataset.set_time_period("2015-07-26") - print(dataset.get_time_period()) - dataset.update_in_hdx() - -14. You can view it on HDX before changing it back (if you have an API key): - - dataset.set_time_period("PREVIOUS DATE") - dataset.update_in_hdx() - -15. If you are storing your data on HDX, you can upload a new file to a - resource: - - resource = dataset.get_resource(0) - resource.set_file_to_upload("PATH TO FILE") - resource.update_in_hdx() - -16. Alternatively, if you are using a URL to point to data held externally from - HDX, you can mark that the data has been updated before updating the - resource or parent dataset: - - resource = dataset.get_resource(2) - resource.mark_data_updated() - dataset.update_in_hdx() - -17. Exit and remove virtualenv: +12. Exit and remove virtualenv: exit() deactivate @@ -131,3 +84,7 @@ virtualenv if not installed: On other OSs: rm -rf test + +To create or update datasets you need an API token and the right to publish for an +organisation. See [Create a Dataset from a CSV](create-a-dataset.md) and +[Obtain your API Key](../how-to/api-key.md). From 8595691b4508fe66146b5247b218fe78505cb16e Mon Sep 17 00:00:00 2001 From: AntoninKns Date: Wed, 11 Nov 2026 21:23:43 +0300 Subject: [PATCH 2/3] Add create a dataset tutorial --- documentation/index.md | 1 + documentation/mkdocs.yaml | 1 + .../assets/bel_example_indicators.csv | 7 + documentation/tutorials/create-a-dataset.md | 205 ++++++++++++++++++ 4 files changed, 214 insertions(+) create mode 100644 documentation/tutorials/assets/bel_example_indicators.csv create mode 100644 documentation/tutorials/create-a-dataset.md diff --git a/documentation/index.md b/documentation/index.md index 716ce5b..b3574c9 100755 --- a/documentation/index.md +++ b/documentation/index.md @@ -11,6 +11,7 @@ upload your datasets to HDX. ## Tutorials - [Read a Dataset](tutorials/getting-started.md) +- [Create a Dataset from a CSV](tutorials/create-a-dataset.md) ## How-to Guides diff --git a/documentation/mkdocs.yaml b/documentation/mkdocs.yaml index 60aee1f..70bc61a 100644 --- a/documentation/mkdocs.yaml +++ b/documentation/mkdocs.yaml @@ -13,6 +13,7 @@ nav: - Home: index.md - Tutorials: - Read a Dataset: tutorials/getting-started.md + - Create a Dataset from a CSV: tutorials/create-a-dataset.md - How-to Guides: - Set up: - Obtain your API Key: how-to/api-key.md diff --git a/documentation/tutorials/assets/bel_example_indicators.csv b/documentation/tutorials/assets/bel_example_indicators.csv new file mode 100644 index 0000000..6c657fb --- /dev/null +++ b/documentation/tutorials/assets/bel_example_indicators.csv @@ -0,0 +1,7 @@ +date,admin1_name,indicator,value +2025-01-31,Brussels,example_index,12.5 +2025-01-31,Flanders,example_index,9.1 +2025-01-31,Wallonia,example_index,10.4 +2025-02-28,Brussels,example_index,13.0 +2025-02-28,Flanders,example_index,9.4 +2025-02-28,Wallonia,example_index,10.2 diff --git a/documentation/tutorials/create-a-dataset.md b/documentation/tutorials/create-a-dataset.md new file mode 100644 index 0000000..23c496e --- /dev/null +++ b/documentation/tutorials/create-a-dataset.md @@ -0,0 +1,205 @@ +# Create a Dataset from a CSV + +In this tutorial, we go from a CSV file to a complete HDX dataset, stopping just before +the upload. Everything up to that point is a dry run that needs no account, so anyone +can follow it. Publishing for real requires an API token and the right to publish for an +organisation on HDX, which is covered at the end. + +The data in this tutorial is made up. Do not upload it to HDX. + +## Before you start + +You need Python 3.11 or later. In a new folder, create and activate a virtual +environment, install the library and start Python: + + python -m venv hdx-tutorial + source hdx-tutorial/bin/activate + pip install hdx-python-api + python + +On Windows, activate the environment with `hdx-tutorial\Scripts\activate` instead. If +you already followed [Read a Dataset](getting-started.md), you can reuse that +environment. + +## 1. Prepare the CSV + +Save the following as **bel_example_indicators.csv** in the folder from which you launch +Python: + + date,admin1_name,indicator,value + 2025-01-31,Brussels,example_index,12.5 + 2025-01-31,Flanders,example_index,9.1 + 2025-01-31,Wallonia,example_index,10.4 + 2025-02-28,Brussels,example_index,13.0 + 2025-02-28,Flanders,example_index,9.4 + 2025-02-28,Wallonia,example_index,10.2 + +CSV is the format HDX recommends for tabular data. HDX also asks that the resource name +and the file name are the same, in lowercase with underscores, and that country files +start with the ISO3 country code. See the +[HDX pipeline recommendations](https://docs.humdata.org/build/hdx-pipeline-recommendations) +for the full conventions, including the limit on file size. + +## 2. Connect to HDX in read-only mode + +Building a dataset needs HDX's lists of approved tags and valid locations, which the +library reads from HDX. A read-only connection to the production server is enough, and +it needs no API key: + + from hdx.utilities.easy_logging import setup_logging + from hdx.api.configuration import Configuration + from hdx.data.dataset import Dataset + + setup_logging() + Configuration.create(hdx_site="prod", user_agent="tutorial_create_dataset", hdx_read_only=True) + +The user agent identifies your code to HDX and must always be supplied. Keep this name +for the tutorial. In your own code, use one that describes your organisation and +project. + +## 3. Build the dataset + + dataset = Dataset( + { + "name": "bel-example-indicators", + "title": "Belgium - Example Indicators", + "notes": "Made-up indicators used to demonstrate dataset creation.", + "dataset_source": "Example source", + "methodology": "Other", + "license_id": "cc-by", + "private": True, + "owner_org": "00000000-0000-0000-0000-000000000000", + "maintainer": "00000000-0000-0000-0000-000000000001", + } + ) + dataset.set_expected_update_frequency("Every month") + dataset.add_country_location("BEL") + dataset.add_tag("economics") + +A few notes on these choices: + +- The dataset name is the end of its web address: lowercase, with hyphens and the ISO3 + code for the location. The title starts with the country name followed by a hyphen + and leaves out dates and the word "data". +- `owner_org` and `maintainer` are the identifiers of an HDX organisation and user. The + zeros are placeholders, because you can only look up real ones with an account. +- Tags must come from HDX's approved list. Try `dataset.add_tag("not-a-real-tag")`: it + returns the tag in its second list, which holds the tags that were rejected, and logs a + warning. +- Avoid the update frequencies "As needed" and "Never" where you can, because HDX uses + this field to judge how up to date a dataset is. See + [Set Expected Update Frequency](../how-to/operate-datasets/update-frequency.md). + +## 4. Generate the resource from the CSV + +`generate_resource` writes the rows to a file and adds a resource to the dataset. Given +the name of the date column, it also sets the time period of the dataset: + + import csv + import tempfile + + with open("bel_example_indicators.csv", newline="", encoding="utf-8") as f: + rows = list(csv.DictReader(f)) + + added, info = dataset.generate_resource( + tempfile.mkdtemp(), + "bel_example_indicators.csv", + rows, + {"name": "bel_example_indicators.csv", "description": "Example indicators"}, + headers=list(rows[0]), + datecol="date", + ) + print(added) + print(dataset.get_time_period()) + print(dataset.get_resources()) + +The time period runs from the earliest date in the column to the end of the day of the +latest date, and the dates are treated as UTC. See +[Generate Resources](../how-to/operate-datasets/resource-generation.md) for the other +ways of generating resources. + +## 5. Validate + +Check that the metadata is complete: + + dataset.check_required_fields() + +Nothing is printed when everything is present. To see what a failure looks like, remove a +field and check again: + + del dataset["dataset_source"] + dataset.check_required_fields() + +This raises `HDXError: Field dataset_source is missing in dataset!`. Put the field back +before continuing: + + dataset["dataset_source"] = "Example source" + +## 6. Look at what would be sent + + print(dataset.data) + print(dataset.get_resources()) + +This is everything the library would send to HDX: the dataset metadata and the resource +with its file. The dry run ends here, and nothing has been written to HDX. + +## 7. Where a dry run stops + +The call that publishes the dataset is: + + dataset.create_in_hdx() + +You need an API token and the right to create datasets for your organisation (editor or +admin). Do not run it in this tutorial: the connection from step 2 has no token, and the +data is made up. + +When a call to HDX fails, the library raises an error whose message hides the reason, +for example `HDXError: Failed when trying to create bel-example-indicators! (POST)`. +The reason is attached to it, and you can print it like this: + + try: + dataset.create_in_hdx() + except Exception as e: + print(repr(e.__cause__)) + +For example, an account with a valid token that does not belong to an organisation was +refused on the HDX demo server with +`Access denied: User not authorised to create packages`. See +[Obtain your API Key](../how-to/api-key.md). + +## 8. For organisation members + +The steps in this section need publishing rights, so they have not been run end to end +for this tutorial. If something here does not work for you, please +[open an issue](https://github.com/OCHA-DAP/hdx-python-api/issues). If you are a member +of an organisation: + +1. Get your API token, as described in [Obtain your API Key](../how-to/api-key.md), and + save it in a file named **.hdx_configuration.yaml** in your home directory. Note the + space after the colon: + + hdx_key: "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" + +2. Replace the placeholder `owner_org` and `maintainer` with real identifiers. For + example: + + from hdx.data.user import User + + organisations = User.get_current_user_organization_dicts("create_dataset") + dataset["owner_org"] = organisations[0]["id"] + dataset["maintainer"] = User.get_current_user()["id"] + +3. Rehearse on a test server before using the production server. Do not use + `hdx_read_only=True`, and choose the site when you create the configuration, for + example `hdx_site="demo"`. A key created on the production site works on `demo` + about a week later, because `demo` is refreshed from production weekly (anything + else on it is lost at that point). The `stage` and `feature` servers are refreshed + rarely, and you must create a separate API key there, which also means asking HDX for + access. + +4. When the rehearsal behaves as expected, run the same code with `hdx_site="prod"`. + Keep `private` set to `True` until you have checked how the dataset looks on HDX. + +Running the same code again later updates the dataset instead of creating a second one. +To schedule it, see [Build a project Framework](../how-to/framework.md) and +[Build a Docker image](../how-to/docker.md). From 8310b3d7b4069eb1691301d3c73079ec31e83fd1 Mon Sep 17 00:00:00 2001 From: AntoninKns Date: Wed, 11 Nov 2026 21:33:35 +0300 Subject: [PATCH 3/3] Add tests to cover tutorial --- documentation/tutorials/create-a-dataset.md | 4 +- .../hdx/data/test_tutorial_create_dataset.py | 93 +++++++++++++++++++ 2 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 tests/hdx/data/test_tutorial_create_dataset.py diff --git a/documentation/tutorials/create-a-dataset.md b/documentation/tutorials/create-a-dataset.md index 23c496e..45d5778 100644 --- a/documentation/tutorials/create-a-dataset.md +++ b/documentation/tutorials/create-a-dataset.md @@ -84,8 +84,8 @@ A few notes on these choices: - `owner_org` and `maintainer` are the identifiers of an HDX organisation and user. The zeros are placeholders, because you can only look up real ones with an account. - Tags must come from HDX's approved list. Try `dataset.add_tag("not-a-real-tag")`: it - returns the tag in its second list, which holds the tags that were rejected, and logs a - warning. + returns the tag in its second list, which holds the tags that were rejected, and logs an + error message. The error is expected here: the tag is dropped and nothing is raised. - Avoid the update frequencies "As needed" and "Never" where you can, because HDX uses this field to judge how up to date a dataset is. See [Set Expected Update Frequency](../how-to/operate-datasets/update-frequency.md). diff --git a/tests/hdx/data/test_tutorial_create_dataset.py b/tests/hdx/data/test_tutorial_create_dataset.py new file mode 100644 index 0000000..1ed37a5 --- /dev/null +++ b/tests/hdx/data/test_tutorial_create_dataset.py @@ -0,0 +1,93 @@ +"""Tests for the code in the "Create a Dataset from a CSV" tutorial""" + +import csv +import json +from datetime import UTC, datetime +from pathlib import Path + +import pytest + +from .test_vocabulary import vocabulary_mockshow +from hdx.api.configuration import Configuration +from hdx.data.dataset import Dataset +from hdx.data.hdxobject import HDXError +from hdx.data.vocabulary import Vocabulary + +tutorial_csv = Path("documentation") / "tutorials" / "assets" +tutorial_csv = tutorial_csv / "bel_example_indicators.csv" + + +class TestTutorialCreateDataset: + @pytest.fixture(scope="function") + def vocabulary_read(self): + Vocabulary._approved_vocabulary = None + Vocabulary._tags_dict = None + + class MockSession: + @staticmethod + def post(url, data, **kwargs): + datadict = json.loads(data.decode("utf-8")) + return vocabulary_mockshow(url, datadict) + + Configuration.read().remoteckan().session = MockSession() + + def build_dataset(self): + dataset = Dataset( + { + "name": "bel-example-indicators", + "title": "Belgium - Example Indicators", + "notes": "Made-up indicators used to demonstrate dataset creation.", + "dataset_source": "Example source", + "methodology": "Other", + "license_id": "cc-by", + "private": True, + "owner_org": "00000000-0000-0000-0000-000000000000", + "maintainer": "00000000-0000-0000-0000-000000000001", + } + ) + dataset.set_expected_update_frequency("Every month") + return dataset + + def test_tutorial_dry_run( + self, configuration, locations, vocabulary_read, tmp_path + ): + dataset = self.build_dataset() + assert dataset.get_expected_update_frequency() == "Every month" + assert dataset.add_country_location("BEL") + assert dataset["groups"] == [{"name": "bel"}] + assert dataset.add_tag("economics") == (["economics"], []) + assert dataset.add_tag("not-a-real-tag") == ([], ["not-a-real-tag"]) + assert dataset.get_tags() == ["economics"] + + with open(tutorial_csv, newline="", encoding="utf-8") as f: + rows = list(csv.DictReader(f)) + added, info = dataset.generate_resource( + tmp_path, + "bel_example_indicators.csv", + rows, + { + "name": "bel_example_indicators.csv", + "description": "Example indicators", + }, + headers=list(rows[0]), + datecol="date", + ) + assert added is True + assert info["startdate"] == datetime(2025, 1, 31, tzinfo=UTC) + assert info["enddate"] == datetime(2025, 2, 28, tzinfo=UTC) + time_period = dataset.get_time_period() + assert time_period["startdate_str"] == "2025-01-31T00:00:00+00:00" + assert time_period["enddate_str"] == "2025-02-28T23:59:59+00:00" + assert [r["name"] for r in dataset.get_resources()] == [ + "bel_example_indicators.csv" + ] + assert (tmp_path / "bel_example_indicators.csv").exists() + + dataset.check_required_fields() + + del dataset["dataset_source"] + with pytest.raises(HDXError, match="Field dataset_source is missing"): + dataset.check_required_fields() + dataset["dataset_source"] = "Example source" + dataset.check_required_fields() + assert "dataset_source" in dataset.data