diff --git a/.github/workflows/test-ceph-custom.yml b/.github/workflows/test-ceph-custom.yml index b9aa65f6c971..7e465f8c3ed6 100644 --- a/.github/workflows/test-ceph-custom.yml +++ b/.github/workflows/test-ceph-custom.yml @@ -4,12 +4,12 @@ on: workflow_dispatch: inputs: base_branch: - description: 'Base branch to test (e.g. squid-19.2.4-cobaltcore-storage)' + description: 'Base branch to test (e.g. tentacle-20.2.4-cobaltcore-storage)' required: true type: string pull_request: branches: - - 'squid-*-cobaltcore-storage' + - 'tentacle-*-cobaltcore-storage' jobs: test-ceph: @@ -49,6 +49,13 @@ jobs: git submodule foreach --recursive git clean -fdx git submodule update --init --recursive + # A job that dies mid-build (e.g. lost runner) leaves its container + # behind, and build-with-container.py always names it ceph_build. + - name: Remove leftover build container + run: | + podman ps -a --filter name=ceph_build + podman rm -f ceph_build + # --- 2. COMPUTE PARALLELISM --- - name: Compute build + test parallelism run: | diff --git a/PendingReleaseNotes b/PendingReleaseNotes index 6ad7159e9798..92cb4298c2ef 100644 --- a/PendingReleaseNotes +++ b/PendingReleaseNotes @@ -879,6 +879,13 @@ CephFS: Disallow delegating preallocated inode ranges to clients. Config * RGW: The serialized format of notification and topics has changed, so that new/updated topics will be unreadable by old RGWs. We recommend completing the RGW upgrades before creating or modifying any notification topics. +* RGW: New rate limiting options have been added to control the number of + bucket listing requests (`radosgw-admin ratelimit set ... --max-list-ops=`) + and delete operations (`radosgw-admin ratelimit set ... --max-delete-ops=`) + per accumulation interval, along with the ability to configure the + accumulation interval duration (`rgw_ratelimit_interval`). + Their default value is `0`, which keeps backward compatibility to the + previous behavior of READ and WRITE operations rate limiting. * RBD: Trailing newline in passphrase files (`` argument in `rbd encryption format` command and `--encryption-passphrase-file` option in other commands) is no longer stripped. diff --git a/doc/radosgw/admin.rst b/doc/radosgw/admin.rst index 1dd411fdcb07..10ef6cb61a41 100644 --- a/doc/radosgw/admin.rst +++ b/doc/radosgw/admin.rst @@ -629,9 +629,21 @@ Rate Limit Management ===================== Quotas can be set for The Ceph Object Gateway on users and buckets. The "rate -limit" includes the maximum number of read operations (read ops) and write -operations (write ops) per minute as well as the number of bytes per minute -that can be written or read per user or per bucket. +limit" includes the maximum number of read operations and write operations +per accumulation interval as well as the number of bytes per accumulation interval +that can be written or read per user or per bucket. It also includes the maximum +number of list requests and delete operations per accumulation interval. +The accumulation interval is configured by the :confval:`rgw_ratelimit_interval` option. +The default value is 60 seconds. +(Note: S3 Multi-Object Delete operation are currently not supported by rate limiting) + +The configured limits should be divided by the number of active object gateways. For example, +if "user A" is to be be limited to 10 ops per minute and there are two object gateways in the cluster, +then the limit on "user A" should be 5 (10 ops per minute / 2 RGWs). +If the requests are not balanced between RGWs, the rate limit might be underutilized. +For example: if the ops limit is 5 and there are two RGWs, +but the Load Balancer sends load to only one of those RGWs, +the effective limit is 5 ops, because this limit is enforced per RGW. Read Requests and Write Requests -------------------------------- @@ -643,8 +655,8 @@ How Metrics Work Each object gateway tracks per-user metrics separately from bucket metrics. These metrics are not shared with other gateways. The configured limits should be divided by the number of active object gateways. For example, if "user A" is -to be be limited to 10 ops per minute and there are two object gateways in the -cluster, then the limit on "user A" should be ``5`` (10 ops per minute / 2 +to be be limited to 10 ops per accumulation interval and there are two object gateways in the +cluster, then the limit on "user A" should be ``5`` (10 ops per accumulation interval / 2 RGWs). If the requests are **not** balanced between RGWs, the rate limit might be underutilized. For example: if the ops limit is ``5`` and there are two RGWs, **but** the Load Balancer sends load to only one of those RGWs, the @@ -661,12 +673,18 @@ rate limit is reached during the execution of the request. The RGW keeps track of a "debt" consisting of bytes used in excess of the configured value; users or buckets that incur this kind of debt are prevented from sending more requests until the "debt" has been repaid. The maximum size of the "debt" is -twice the max-read/write-bytes per minute. If "user A" is subject to a 1-byte -read limit per minute and they attempt to ``GET`` an object that is 1 GB in size, +twice the max-read/write-bytes per accumulation interval. If "user A" is subject to a 1-byte +read limit per accumulation interval and they attempt to ``GET`` an object that is 1 GB in size, then the ``GET`` action will fail. After "user A" has completed this 1 GB -operation, RGW blocks the user's requests for up to two minutes. After this +operation, RGW blocks the user's requests for up to two accumulation intervals. After this time has elapsed, "user A" will be able to send ``GET`` requests again. +Rate-limited requests receive an HTTP ``503`` response with the S3 error code +``SlowDown``, matching the AWS S3 behavior. The response includes a +``Retry-After`` header with the estimated number of seconds until the rate-limit +window allows the next request through, computed from the current token-bucket +state. Load balancers or reverse proxies can be configured to map the ``503`` +with ``Retry-After`` to ``429 Too Many Requests`` if clients require it. - **Bucket:** The ``--bucket`` option allows you to specify a rate limit for a bucket. @@ -675,19 +693,27 @@ time has elapsed, "user A" will be able to send ``GET`` requests again. user. - **Maximum Read Ops:** The ``--max-read-ops`` setting allows you to limit read - bytes per minute per RGW instance. A ``0`` value disables throttling. + bytes per accumulation interval per RGW instance. A ``0`` value disables throttling. - **Maximum Read Bytes:** The ``--max-read-bytes`` setting allows you to limit - read bytes per minute per RGW instance. A ``0`` value disables throttling. + read bytes per accumulation interval per RGW instance. A ``0`` value disables throttling. - **Maximum Write Ops:** The ``--max-write-ops`` setting allows you to specify - the maximum number of write ops per minute per RGW instance. A ``0`` value + the maximum number of write ops per accumulation interval per RGW instance. A ``0`` value disables throttling. - **Maximum Write Bytes:** The ``--max-write-bytes`` setting allows you to - specify the maximum number of write bytes per minute per RGW instance. A + specify the maximum number of write bytes per accumulation interval per RGW instance. A ``0`` value disables throttling. - + +- **Maximum List Ops:** The ``--max-list-ops`` setting allows you to + specify the maximum number of bucket listing requests per accumulation interval per RGW instance. + A ``0`` value disables throttling. + +- **Maximum Delete Ops:** The ``--max-delete-ops`` setting allows you to + specify the maximum number of delete operations per accumulation interval per RGW instance. + A ``0`` value disables throttling. + - **Rate Limit Scope:** The ``--ratelimit-scope`` option sets the scope for the rate limit. The options are ``bucket`` , ``user`` and ``anonymous``. Bucket rate limit apply to buckets. The user rate limit applies to a user. The @@ -706,7 +732,8 @@ parameters: radosgw-admin ratelimit set --ratelimit-scope=user --uid= \ <[--max-read-ops=] [--max-read-bytes=] \ - [--max-write-ops=] [--max-write-bytes=]> + [--max-write-ops=] [--max-write-bytes=] \ + [--max-list-ops=] [--max-delete-ops=]> An example of using ``radosgw-admin ratelimit set`` to set a rate limit might look like this: @@ -771,7 +798,8 @@ The following is the general form of commands that set rate limit parameters: radosgw-admin ratelimit set --ratelimit-scope=bucket --bucket= \ <[--max-read-ops=] [--max-read-bytes=] \ - [--max-write-ops=] [--max-write-bytes=]> + [--max-write-ops=] [--max-write-bytes=] \ + [--max-list-ops=] [--max-delete-ops=]> An example of using ``radosgw-admin ratelimit set`` to set a rate limit for a bucket might look like this: diff --git a/doc/radosgw/adminops.rst b/doc/radosgw/adminops.rst index 0433b2bb104c..569591586d4d 100644 --- a/doc/radosgw/adminops.rst +++ b/doc/radosgw/adminops.rst @@ -2788,8 +2788,9 @@ permission. :: Rate Limit ========== -The Admin Operations API enables you to set and get ratelimit configurations on users and on bucket and global rate limit configurations. See `Rate Limit Management`_ for additional details. -Rate Limit includes the maximum number of operations and/or bytes per minute, separated by read and/or write, to a bucket and/or by a user and the maximum storage size in megabytes. +The Admin Operations API enables you to set and get ratelimit configurations on users and on bucket and global rate limit configurations. See `Rate Limit Management`_ for additional details. +Rate Limit includes the maximum number of operations and/or bytes per accumulation interval, separated by read and/or write (Additionally list and get operations), +to a bucket and/or by a user and the maximum storage size in megabytes. To view rate limit, the user must have a ``ratelimit=read`` capability. To set, modify or disable a ratelimit, the user must have ``ratelimit=write`` capability. @@ -2803,16 +2804,22 @@ Valid parameters for quotas include: - **User:** The ``uid`` option allows you to specify a rate limit for a user. - **Maximum Read Bytes:** The ``max-read-bytes`` setting allows you to specify - the maximum number of read bytes per minute. A 0 value disables this setting. + the maximum number of read bytes per accumulation interval. A 0 value disables this setting. - **Maximum Write Bytes:** The ``max-write-bytes`` setting allows you to specify - the maximum number of write bytes per minute. A 0 value disables this setting. + the maximum number of write bytes per accumulation interval. A 0 value disables this setting. - **Maximum Read Ops:** The ``max-read-ops`` setting allows you to specify - the maximum number of read ops per minute. A 0 value disables this setting. + the maximum number of read ops per accumulation interval. A 0 value disables this setting. - **Maximum Write Ops:** The ``max-write-ops`` setting allows you to specify - the maximum number of write ops per minute. A 0 value disables this setting. + the maximum number of write ops per accumulation interval. A 0 value disables this setting. + +- **Maximum List Ops:** The ``max-list-ops`` setting allows you to specify + the maximum number of bucket listing requests per accumulation interval. A 0 value disables this setting. + +- **Maximum Delete Ops:** The ``max-delete-ops`` setting allows you to specify + the maximum number of delete operations per accumulation interval. A 0 value disables throttling. - **Global:** The ``global`` option allows you to specify a global rate limit. The value should be either 'True' or 'False'. @@ -2839,7 +2846,7 @@ Set User Rate Limit To set a rate limit, the user must have ``ratelimit`` capability set with ``write`` permission. :: - POST /{admin}/ratelimit?ratelimit-scope=user&uid=<[&max-read-bytes=][&max-write-bytes=][&max-read-ops=][&max-write-ops=][enabled=]> + POST /{admin}/ratelimit?ratelimit-scope=user&uid=<[&max-read-bytes=][&max-write-bytes=][&max-read-ops=][&max-write-ops=][&max-list-ops=][&max-delete-ops=][&enabled=]> @@ -2859,7 +2866,7 @@ Set Rate Limit for an Individual Bucket To set a rate limit, the user must have ``ratelimit`` capability set with ``write`` permission. :: - POST /{admin}/ratelimit?bucket=&ratelimit-scope=bucket<[&max-read-bytes=][&max-write-bytes=][&max-read-ops=][&max-write-ops=]> + POST /{admin}/ratelimit?bucket=&ratelimit-scope=bucket<[&max-read-bytes=][&max-write-bytes=][&max-read-ops=][&max-write-ops=][&max-list-ops=][&max-delete-ops=][&enabled=]> @@ -2879,7 +2886,7 @@ Set Global User Rate Limit To set a rate limit, the user must have ``ratelimit`` capability set with ``write`` permission. :: - POST /{admin}/ratelimit?ratelimit-scope=user&global=<[&max-read-bytes=][&max-write-bytes=][&max-read-ops=][&max-write-ops=][enabled=]> + POST /{admin}/ratelimit?ratelimit-scope=user&global=<[&max-read-bytes=][&max-write-bytes=][&max-read-ops=][&max-write-ops=][&max-list-ops=][&max-delete-ops=][&enabled=]> @@ -2889,7 +2896,7 @@ Set Global Rate Limit Bucket To set a rate limit, the user must have ``ratelimit`` capability set with ``write`` permission. :: - POST /{admin}/ratelimit?ratelimit-scope=bucket&global=<[&max-read-bytes=][&max-write-bytes=][&max-read-ops=][&max-write-ops=]> + POST /{admin}/ratelimit?ratelimit-scope=bucket&global=<[&max-read-bytes=][&max-write-bytes=][&max-read-ops=][&max-write-ops=][&max-list-ops=][&max-delete-ops=][&enabled=]> @@ -2899,7 +2906,7 @@ Set Global Anonymous User Rate Limit To set a rate limit, the user must have ``ratelimit`` capability set with ``write`` permission. :: - POST /{admin}/ratelimit?ratelimit-scope=anon&global=<[&max-read-bytes=][&max-write-bytes=][&max-read-ops=][&max-write-ops=][enabled=]> + POST /{admin}/ratelimit?ratelimit-scope=anon&global=<[&max-read-bytes=][&max-write-bytes=][&max-read-ops=][&max-write-ops=][&max-list-ops=][&max-delete-ops=][&enabled=]> diff --git a/doc/radosgw/bucketpolicy.rst b/doc/radosgw/bucketpolicy.rst index 23356624a46a..0ade1b080f1c 100644 --- a/doc/radosgw/bucketpolicy.rst +++ b/doc/radosgw/bucketpolicy.rst @@ -122,6 +122,11 @@ For all requests, condition keys we support are: - aws:UserAgent - aws:username +Request that authenticate with Keystone also include: + +- keystone:role +- keystone:userid + We support certain s3 condition keys for bucket and object requests. *Support for the following bucket-related operations was added in the Mimic diff --git a/doc/radosgw/keystone.rst b/doc/radosgw/keystone.rst index 068eac24ff2c..21f79e490599 100644 --- a/doc/radosgw/keystone.rst +++ b/doc/radosgw/keystone.rst @@ -159,6 +159,19 @@ S3 API (with AWS-like access and secret keys), if the ``rgw s3 auth use keystone`` option is set. For details, see :doc:`s3/authentication`. +Requests authenticated via Keystone (Swift tokens or S3 with Keystone-managed +credentials) expose the Keystone idenitity in the IAM policy environment +as the condition keys: ``keystone:role`` (role names) and +``keystone:userid`` (user UUID). These conditions can be used in bucket +policies and idenitity policies with ``StringEquals``, ``StringNotEquals`` etc. + +- **keystone:role** - Allow or deny by *role* (e.g only users with role + ``reader`` get read access). Use for RBAC. +- **keystone:userid** - Restrict to specific *user*. Use when policy + depends on a specific user, not just their role. + +See :doc:`bucketpolicy` for list of supported condition keys. + Service token support --------------------- diff --git a/qa/suites/rados/upgrade/cephx/X/ms_bind/.qa b/qa/suites/rados/upgrade/cephx/X/ms_bind/.qa new file mode 120000 index 000000000000..a602a0353e75 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/X/ms_bind/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/rados/upgrade/cephx/X/tasks/07-client-auth-keys/.qa b/qa/suites/rados/upgrade/cephx/X/tasks/07-client-auth-keys/.qa new file mode 120000 index 000000000000..a602a0353e75 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/X/tasks/07-client-auth-keys/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/rados/upgrade/cephx/X/tasks/08-allowed-ciphers/.qa b/qa/suites/rados/upgrade/cephx/X/tasks/08-allowed-ciphers/.qa new file mode 120000 index 000000000000..a602a0353e75 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/X/tasks/08-allowed-ciphers/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/rados/upgrade/cephx/release-X/ms_bind/.qa b/qa/suites/rados/upgrade/cephx/release-X/ms_bind/.qa new file mode 120000 index 000000000000..a602a0353e75 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/release-X/ms_bind/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/rados/upgrade/cephx/release-X/tasks/07-client-auth-keys/.qa b/qa/suites/rados/upgrade/cephx/release-X/tasks/07-client-auth-keys/.qa new file mode 120000 index 000000000000..a602a0353e75 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/release-X/tasks/07-client-auth-keys/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/rados/upgrade/cephx/release-X/tasks/08-allowed-ciphers/.qa b/qa/suites/rados/upgrade/cephx/release-X/tasks/08-allowed-ciphers/.qa new file mode 120000 index 000000000000..a602a0353e75 --- /dev/null +++ b/qa/suites/rados/upgrade/cephx/release-X/tasks/08-allowed-ciphers/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/tasks/ceph.py b/qa/tasks/ceph.py index 81dbdca2d498..84bdbeaaf27f 100644 --- a/qa/tasks/ceph.py +++ b/qa/tasks/ceph.py @@ -1725,8 +1725,6 @@ def key_prune(ctx, config): elif isinstance(config, list): config = {'keys': config} - testdir = teuthology.get_testdir(ctx) - cluster_name = config.setdefault('cluster', 'ceph') manager = ctx.managers[cluster_name] @@ -1789,7 +1787,6 @@ def key_rotate(ctx, config): ] daemons = ctx.daemons.resolve_role_list(config.get('daemons', None), CEPH_ROLE_TYPES, True) - clusters = set() new_mon_key = None for role in daemons: @@ -1831,7 +1828,6 @@ def key_rotate(ctx, config): daemon.restart() clients = config.get('clients', []) - firstmon = teuthology.get_first_mon(ctx, config, cluster_name) keyring = ctx.ceph[cluster_name].keyring for client in clients: if client == 'all': @@ -2272,7 +2268,6 @@ def task(ctx, config): # stop logging health to clog during shutdown, or else we generate # a bunch of scary messages unrelated to our actual run. - firstmon = teuthology.get_first_mon(ctx, config, cluster_name) ctx.ceph[cluster_name].admin.run( args=[ 'sudo', diff --git a/qa/tasks/radosbench.py b/qa/tasks/radosbench.py index 825b5c9c76b5..7bd97ea82fcd 100644 --- a/qa/tasks/radosbench.py +++ b/qa/tasks/radosbench.py @@ -110,7 +110,7 @@ def task(ctx, config): extra_args.append(f'--auth_exit_on_failure={auth_exit_on_failure}') bench_args = [ - f'bench', + 'bench', f'--concurrent-ios={concurrency}', ] @@ -163,7 +163,7 @@ def task(ctx, config): log.info('joining radosbench (timing out after %ss)', timeout) try: run.wait(radosbench.values(), timeout=timeout) - except CommandFailedError as e: + except CommandFailedError: for p in radosbench.values(): if p.exitstatus == expected_rc: pass diff --git a/qa/tasks/radosgw_admin_rest.py b/qa/tasks/radosgw_admin_rest.py index 3a64604d5f5d..3cc12fd568e7 100644 --- a/qa/tasks/radosgw_admin_rest.py +++ b/qa/tasks/radosgw_admin_rest.py @@ -1072,3 +1072,74 @@ def task(ctx, config): # checking remove account access (ret, out) = rgwadmin_rest(endpoint, admin_creds, ['account', 'rm'], {'id' : account_id}) assert ret == 200 + + # TESTCASE 'ratelimit' 'user' 'modify' 'max-list-ops = 100' 'succeeds' + (ret, out) = rgwadmin_rest(endpoint, admin_creds, ['ratelimit', 'modify'], {'ratelimit-scope' : 'user', 'uid' : ratelimit_user, 'max-list-ops' : '100'}) + assert ret == 200 + (ret, out) = rgwadmin_rest(endpoint, admin_creds, ['ratelimit', 'info'], {'ratelimit-scope' : 'user', 'uid' : ratelimit_user}) + assert ret == 200 + user_ratelimit = out['user_ratelimit'] + assert user_ratelimit['max_list_ops'] == 100 + + # TESTCASE 'ratelimit' 'user' 'modify' 'max-delete-ops = 50' 'succeeds' + (ret, out) = rgwadmin_rest(endpoint, admin_creds, ['ratelimit', 'modify'], {'ratelimit-scope' : 'user', 'uid' : ratelimit_user, 'max-delete-ops' : '50'}) + assert ret == 200 + (ret, out) = rgwadmin_rest(endpoint, admin_creds, ['ratelimit', 'info'], {'ratelimit-scope' : 'user', 'uid' : ratelimit_user}) + assert ret == 200 + user_ratelimit = out['user_ratelimit'] + assert user_ratelimit['max_delete_ops'] == 50 + + # TESTCASE 'ratelimit' 'bucket' 'modify' 'max-list-ops = 200' 'succeeds' + (ret, out) = rgwadmin_rest(endpoint, admin_creds, ['ratelimit', 'modify'], {'ratelimit-scope' : 'bucket', 'bucket' : ratelimit_bucket, 'max-list-ops' : '200'}) + assert ret == 200 + (ret, out) = rgwadmin_rest(endpoint, admin_creds, ['ratelimit', 'info'], {'ratelimit-scope' : 'bucket', 'bucket' : ratelimit_bucket}) + assert ret == 200 + bucket_ratelimit = out['bucket_ratelimit'] + assert bucket_ratelimit['max_list_ops'] == 200 + + # TESTCASE 'ratelimit' 'bucket' 'modify' 'max-delete-ops = 75' 'succeeds' + (ret, out) = rgwadmin_rest(endpoint, admin_creds, ['ratelimit', 'modify'], {'ratelimit-scope' : 'bucket', 'bucket' : ratelimit_bucket, 'max-delete-ops' : '75'}) + assert ret == 200 + (ret, out) = rgwadmin_rest(endpoint, admin_creds, ['ratelimit', 'info'], {'ratelimit-scope' : 'bucket', 'bucket' : ratelimit_bucket}) + assert ret == 200 + bucket_ratelimit = out['bucket_ratelimit'] + assert bucket_ratelimit['max_delete_ops'] == 75 + + # TESTCASE 'ratelimit' 'global' 'modify' 'bucket' 'max-list-ops = 500' 'succeeds' + (ret, out) = rgwadmin_rest(endpoint, admin_creds, ['ratelimit', 'modify'], {'ratelimit-scope' : 'bucket', 'global': 'true', 'max-list-ops' : '500'}) + assert ret == 200 + (ret, out) = rgwadmin_rest(endpoint, admin_creds, ['ratelimit', 'info'], {'global' : 'true'}) + assert ret == 200 + # Check that global bucket ratelimit has the list ops set + assert 'bucket_ratelimit' in out + assert out['bucket_ratelimit']['max_list_ops'] == 500 + + # TESTCASE 'ratelimit' 'global' 'modify' 'bucket' 'max-delete-ops = 300' 'succeeds' + (ret, out) = rgwadmin_rest(endpoint, admin_creds, ['ratelimit', 'modify'], {'ratelimit-scope' : 'bucket', 'global': 'true', 'max-delete-ops' : '300'}) + assert ret == 200 + (ret, out) = rgwadmin_rest(endpoint, admin_creds, ['ratelimit', 'info'], {'global' : 'true'}) + assert ret == 200 + # Check that global bucket ratelimit has the delete ops set + assert 'bucket_ratelimit' in out + assert out['bucket_ratelimit']['max_delete_ops'] == 300 + + # TESTCASE 'ratelimit' 'user' 'modify' 'multiple ops at once' 'succeeds' + (ret, out) = rgwadmin_rest(endpoint, admin_creds, ['ratelimit', 'modify'], { + 'ratelimit-scope' : 'user', + 'uid' : ratelimit_user, + 'max-read-ops' : '1000', + 'max-write-ops' : '800', + 'max-list-ops' : '600', + 'max-delete-ops' : '400', + 'enabled' : 'true' + }) + assert ret == 200 + (ret, out) = rgwadmin_rest(endpoint, admin_creds, ['ratelimit', 'info'], {'ratelimit-scope' : 'user', 'uid' : ratelimit_user}) + assert ret == 200 + user_ratelimit = out['user_ratelimit'] + assert user_ratelimit['enabled'] == True + assert user_ratelimit['max_read_ops'] == 1000 + assert user_ratelimit['max_write_ops'] == 800 + assert user_ratelimit['max_list_ops'] == 600 + assert user_ratelimit['max_delete_ops'] == 400 + diff --git a/release-management/backports/01a11a76-84c1-7371-840f-b82c5fbf1a72.md b/release-management/backports/01a11a76-84c1-7371-840f-b82c5fbf1a72.md new file mode 100644 index 000000000000..19baad17f0fa --- /dev/null +++ b/release-management/backports/01a11a76-84c1-7371-840f-b82c5fbf1a72.md @@ -0,0 +1,144 @@ +--- +id: 01a11a76-84c1-7371-840f-b82c5fbf1a72 +title: rate limit "retry after" header + "rgw_ratelimit_interval" change +component: rgw +category: feature +provenance: + type: upstream_pr + upstream_prs: + - https://github.com/ceph/ceph/pull/64762 + - https://github.com/ceph/ceph/pull/66509 + - https://github.com/ceph/ceph/pull/68210 + upstream_pr_state: merged-main + upstream_commits: + - dae572d50080609c77d7131cfc99b1fb3f16d31b + - 91e9f00a7543c4651a983c2d785696a069d8ab28 + - 3dae903c9a31fe5161058a4bb1cfe4092839eb29 + upstream_trackers: + - https://tracker.ceph.com/issues/72894 + - https://tracker.ceph.com/issues/74088 + - https://tracker.ceph.com/issues/64083 + - https://tracker.ceph.com/issues/67655 +justification: + type: customer + refs: + - cobaltcore-dev/cloud-storage#588 + - todo + - '```' + - https://github.com/ceph/ceph/commit/3dae903c9a31fe5161058a4bb1cfe4092839eb29 + - https://tracker.ceph.com/issues/64083 + - https://tracker.ceph.com/issues/67655 + - 'plus:' + - ceph/ceph#64762 + - See also the squid backport effort for retry after that includes the + - ratelimit_interval change + - https://github.com/cobaltcore-dev/ceph/pull/19/commits + +risk: + # Stage-B — fill these during PR prep. See release-management/backports/RISK-RUBRIC.md. + blast: availability + conflict: trivial + coverage: partial +data_structure_change: + struct_v_bump: RGWRateLimitInfo 1 -> 2 + upstream_coordinated: true +--- + +## Justification (public) + +Clients rejected by RGW rate limiting receive no guidance on when to retry, so +they either back off far longer than necessary or hammer the endpoint until +they succeed. This backport adds a `Retry-After` header to rate-limited +responses, computed from the client's actual token deficit and replenishment +rate. + +It also adds rate limiting for LIST and DELETE operations, which previously +could not be limited independently of general read and write traffic, and makes +the token bucket accumulation window configurable via `rgw_ratelimit_interval`. + +dditionally, the backport fixes a bug. With rate limiting enabled but an +individual limit left at 0 (unlimited), ops tokens are still deducted per +request while never being refilled. That is harmless on its own, because the +credit isn't checked while the limit is 0. But if the operator later sets that +limit, the credit may already be far negative and recovers only at the newly +onfigured rate, so the user can be blocked for a long time. + +## Justification (internal) + +Three upstream changes are part of the backport: + +1. **ceph/ceph#64762** — LIST/DELETE rate limiting, `rgw_ratelimit_interval`. +2. **ceph/ceph#66509** — fixes unbounded negative ops budget. +3. **ceph/ceph#68210** — adds the `Retry-After` header itself. + +PR 1. and 3. are required by the customer. 2. is needed addition to 1. because +it resolves a bug and omits merge conflicts. + +## Risk notes + +**Blast: availability.** +- The backport changes the request admission. A defect is more likely to change + how traffic is accepted or rejected than to corrupt data. +- `dae572` bumps `RGWRateLimitInfo` from struct_v 1 to 2 to add `max_list_ops` + and `max_delete_ops`. Reading old records with the new build is safe: the + `struct_v >= 2` guards default both new fields to `0`, so v1 records decode + and behave as before. + The other direction is not. `compat_v` stays at `1` while the new fields are + inserted in the middle of the field order. So an old decoder doesn't reject a + v2 record. It reads it positionally and shifts the fields: `max_list_ops` + ends up in `max_write_bytes`, `max_delete_ops` in `max_read_bytes`, and + `enabled` gets the low byte of the real `max_write_bytes`. An RGW on the old + build reading a record written with `max_list_ops = 100` applies a 100 byte + write bandwidth limit. There is no decode error, just wrong values. + v2 records only appear once a rate limit is written by the new build + (`radosgw-admin ratelimit set` or the admin REST API). The upgrade itself + does not rewrite stored records, so an upgrade without a config change + produces no v2 data. +- Additionally, there is a minor issue the backport (`91e9f00`) introduces. If + the user is charged for a request and the bucket rejects the request, the + user gets the token back. The problem: it also happens if the user has + unlimited access. + So the amount of credits for a user can grow over time. However, it doesn't + matter much because it's corrected by increase_tokens, min(max * 1000, …) on + the next refill. Compared to the issue the commit is fixing, it's a minor one. + +**Conflict: trivial.** `91e9f00` and `3dae903` both cherry-pick onto the branch +with no conflicts at all. Only `dae572` was cherry-picked with minor conflicts +that are documented as part of the commit message. + +**Coverage: partial.** +- `91e9f00` adds four regression tests + (`unlimited_access_not_left_large_*_ops_budget`) +- `3dae903` adds + `TEST(RGWRateLimitEntry, compute_delay)` and updates the existing suite to + the new `int64_t` return +- `dae572` contributed both unit and integration tests for the new admin + options. + +No test covers the refund asymmetry described above. + +## Behavior change + +- Rate-limited S3 responses now carry a `Retry-After` header giving the seconds + until the client's credit recovers. The status code is unchanged, still + `503` (Swift: `498`). Operators who need `429` must remap at the proxy layer. +- New rate limit dimensions `max_list_ops` and `max_delete_ops` can be set via + `radosgw-admin ratelimit set` and the admin REST API. Both default to `0` + (unlimited). +- New config option `rgw_ratelimit_interval` (default `60` seconds) controls + the token bucket accumulation window. Previously hardcoded at 60, so it stays + the same in case of the default value. +- Ops counters no longer accumulate unbounded debt while a limit is `0`. A user + who would previously have been rejected for hours or days on first enabling a + limit now starts from a clean budget. + +## Upgrade notes + +`rgw_ratelimit_interval` is flagged `startup`, so changing it from the default +requires an RGW restart. + +Don't change any rate limit configuration until all RGWs run the new build.`RGW_ATTR_RATELIMIT` is shared state read by every RGW, radosgw-admin and the +admin REST API. A `ratelimit set` during the rollout rewrites the record as v2 +and any RGW still on the old build misreads it (see Risk notes). Reads are +unaffected, so the rolling restart itself is safe, only writes create v2 +records. diff --git a/release-management/backports/01a12f91-ee54-7edc-80b3-610fd82571ba.md b/release-management/backports/01a12f91-ee54-7edc-80b3-610fd82571ba.md new file mode 100644 index 000000000000..62d24b6659be --- /dev/null +++ b/release-management/backports/01a12f91-ee54-7edc-80b3-610fd82571ba.md @@ -0,0 +1,46 @@ +--- +id: 01a12f91-ee54-7edc-80b3-610fd82571ba +title: Fix Existing Build Errors +component: build +category: fix +provenance: + type: upstream_pr + upstream_prs: + - https://github.com/ceph/ceph/pull/71165 + - https://github.com/ceph/ceph/pull/71190 + upstream_commits: + - 6b206bc281c05b773b3e24c11b248a45c86ffed4 + - a8eddc15b6a7a6ddad7bc4cac27af2022cb3999b + upstream_pr_state: merged-stable + previous_backport: 01a0c326-e041-718b-bf29-f10dc5260620 +justification: + type: engineering + refs: + - cobaltcore-dev/cloud-storage#601 +risk: + # blast + coverage inherited from 01a0c326-e041-718b-bf29-f10dc5260620 (prior effort). + blast: availability + # conflict is a property of (change x base) — re-score against + # the new base; the schema rejects REPLACE_ME until you do. + conflict: clean + coverage: strong +--- + +## Justification (public) + +custom github build process uses -Werror and is failing. This errors are fixed in upstream after release 20.2.4 + +## Justification (internal) + +custom github build process uses -Werror and is failing. This errors are fixed in upstream after release 20.2.4 + +## Risk notes + +fixes warnings that will be treaten as errors during build for unit tests + +## Behavior change + + +## Upgrade notes + +No configuration, data migration, restart sequencing, or ACL rewrite is needed. diff --git a/release-management/backports/01a16411-8df8-7e1f-baaf-8aa40a09ecdd.md b/release-management/backports/01a16411-8df8-7e1f-baaf-8aa40a09ecdd.md new file mode 100644 index 000000000000..2b36cd87152c --- /dev/null +++ b/release-management/backports/01a16411-8df8-7e1f-baaf-8aa40a09ecdd.md @@ -0,0 +1,80 @@ +--- +id: 01a16411-8df8-7e1f-baaf-8aa40a09ecdd +title: 'IAM / Bucket Policy: Keystone role and user ID support' +component: rgw +category: feature +provenance: + type: upstream_pr + upstream_prs: + - https://github.com/ceph/ceph/pull/67752 + upstream_pr_state: merged-main + upstream_commits: + - b06b7a82ed1a5c57dce8b9ce2fa0931fe0f55ff7 + previous_backport: 01a0cdb3-3f84-74ab-9403-c7638a039697 +justification: + type: customer + refs: + - cobaltcore-dev/cloud-storage#566 + - cobaltcore-dev/cloud-storage#565 + - https://github.com/cobaltcore-dev/cloud-storage/issues/380 + - https://github.com/cobaltcore-dev/cloud-storage/issues/372 +risk: + blast: cosmetic + conflict: trivial + coverage: strong +--- + +## Justification (public) + +Allows Keystone role names and user IDs to be injected as `keystone:role` and +`keystone:userid` condition keys in S3 bucket policies, enabling role-based and +per-user access control via standard IAM policy conditions. + +## Justification (internal) + +Improves keystone authorization features. + +## Risk notes + +**blast: cosmetic** — purely additive. Two new condition keys are populated for +Keystone-authenticated requests. No existing condition key, policy or auth path +changes meaning, and non-Keystone requests are untouched. + +**conflict: trivial** + +**coverage: strong** — +47/47 unit tests pass in `unittest_rgw_iam_policy`. +14/14 live checks pass against vstart RGW and a real Keystone, with two +multi-role users in one project: `keystone:role` matching (single value, +multi-value, shared role, absent role), case handling (`StringEquals` is +case-sensitive, `StringEqualsIgnoreCase` is not), `keystone:userid` matching per +user, `StringLike` wildcards, and three `StringNotEquals` cases that confirm the +AND semantics with multiple roles. +7/7 live grant checks pass with non-owner users from a second project: `Allow` +on `keystone:role` or `keystone:userid` grants exactly the listed read, list, +write and delete actions, users without the role or ID get nothing, and `Deny` +overrides `Allow`. + +## Behavior change + +Requests authenticated via Keystone now expose two additional IAM policy +condition keys: `keystone:role` (one entry per role, lowercased) and +`keystone:userid` (the Keystone user UUID). Requests not authenticated via +Keystone are unchanged, and no existing policy changes behaviour. + +### Two points for policy authors + +Role names in policies are case-sensitive. Roles are lowercased on the way into +the policy environment, so how the role is spelled in Keystone does not matter, +but the value typed in the policy is compared exactly. A policy containing +`"TestRole"` never matches and gives no warning. Write role names in lower case, +or use `StringEqualsIgnoreCase`. + +Rules that distinguish between users of the same project must be written as +`Deny`. All users in an OpenStack project share a single RGW account, so each is +an owner of buckets created there, and owners are not restricted by `Allow` +conditions. `Deny` statements do apply to owners and discriminate correctly. + +## Upgrade notes + +None. diff --git a/src/cephadm/cephadmlib/daemons/ceph.py b/src/cephadm/cephadmlib/daemons/ceph.py index 8d4bed0e9f24..b05bfaac198c 100644 --- a/src/cephadm/cephadmlib/daemons/ceph.py +++ b/src/cephadm/cephadmlib/daemons/ceph.py @@ -339,7 +339,8 @@ def rotate_osd_lv_keyring( osd_lv_data = osd_bluestore_data.get(self.identity.daemon_id, []) if not osd_lv_data: logger.info( - 'No lv data found for OSD.%s, checking for raw OSD', self.identity.daemon_id + 'No lv data found for OSD.%s, checking for raw OSD', + self.identity.daemon_id, ) c_v_container = CephContainer( ctx, @@ -363,8 +364,13 @@ def rotate_osd_lv_keyring( ) raw_list_data = json.loads(out) for osd_data in raw_list_data.values(): - if str(osd_data.get('osd_id', -1)) == str(self.identity.daemon_id): - logger.info('Found data for osd.%s in raw list', str(self.identity.daemon_id)) + if str(osd_data.get('osd_id', -1)) == str( + self.identity.daemon_id + ): + logger.info( + 'Found data for osd.%s in raw list', + str(self.identity.daemon_id), + ) # misnomer since this isn't an LV, but it is the path # we need to pass to the bluestore tool lv_path = osd_data.get('device', None) @@ -476,10 +482,14 @@ def rotate_osd_lv_keyring( f'Out:{out}\n' f'Err:{err}' ) - logger.info(f'Got issue rotating osd keyring using ceph-bluestore-tool:\n{out}\n{err}\nRetrying in {i} seconds') + logger.info( + f'Got issue rotating osd keyring using ceph-bluestore-tool:\n{out}\n{err}\nRetrying in {i} seconds' + ) time.sleep(i) else: - logger.info(f'Successfully rotated osd.{self.identity.daemon_id} keyring') + logger.info( + f'Successfully rotated osd.{self.identity.daemon_id} keyring' + ) break if encrypted: @@ -508,8 +518,11 @@ def rotate_osd_lv_keyring( f'Err:{err}' ) - call(ctx, ['systemctl', 'reset-failed', self.identity.unit_name], - verbosity=CallVerbosity.QUIET_UNLESS_ERROR) + call( + ctx, + ['systemctl', 'reset-failed', self.identity.unit_name], + verbosity=CallVerbosity.QUIET_UNLESS_ERROR, + ) call( ctx, ['systemctl', 'start', self.identity.unit_name], diff --git a/src/common/entity_name.cc b/src/common/entity_name.cc index 895f32f2b6fe..1cc46fa4dbb2 100644 --- a/src/common/entity_name.cc +++ b/src/common/entity_name.cc @@ -149,7 +149,7 @@ std::string EntityName::get_valid_types_as_str() { if (i > 0) { out << ", "; } - out << STR_TO_ENTITY_TYPE[i].first; + out << STR_TO_ENTITY_TYPE[i].second; } return out.str(); } diff --git a/src/common/options/rgw.yaml.in b/src/common/options/rgw.yaml.in index 24d451d77143..d5c1983e1210 100644 --- a/src/common/options/rgw.yaml.in +++ b/src/common/options/rgw.yaml.in @@ -4508,3 +4508,18 @@ options: flags: - runtime with_legacy: true + +- name: rgw_ratelimit_interval + type: uint + level: advanced + desc: Time window for rate limiting in seconds + long_desc: This option sets the time window for rate limiting accumulation in seconds. + Requests that exceed the configured rate limits within this time window will be rejected. + The default is a 60 second token bucket. + default: 60 + min: 1 + services: + - rgw + flags: + - startup + with_legacy: true diff --git a/src/mon/MonClient.h b/src/mon/MonClient.h index 2e0c739f4b42..27ebc0e873f9 100644 --- a/src/mon/MonClient.h +++ b/src/mon/MonClient.h @@ -64,7 +64,7 @@ class MonConnection { ceph::mutex& m); ~MonConnection(); MonConnection(MonConnection&& rhs) = default; - MonConnection& operator=(MonConnection&&) = default; + MonConnection& operator=(MonConnection&&) = delete; MonConnection(const MonConnection& rhs) = delete; MonConnection& operator=(const MonConnection&) = delete; int handle_auth(MAuthReply *m, diff --git a/src/pybind/mgr/cephadm/upgrade.py b/src/pybind/mgr/cephadm/upgrade.py index 090c95fbf08e..7454242e3eab 100644 --- a/src/pybind/mgr/cephadm/upgrade.py +++ b/src/pybind/mgr/cephadm/upgrade.py @@ -1081,6 +1081,12 @@ def _rotate_mgr_mon_auth_keys(self, target_image: str, target_digests: Optional[ self.upgrade_state.rotated_mgr_mon_auth_key_daemons = [] # do mgr and mon keyrings as one off after mons have been upgraded mon_daemons = self.mgr.cache.get_daemons_by_service('mon') + if not mon_daemons: + # Without any mon daemons in the cache we cannot tell whether + # the mons have been upgraded, and there is no mon keyring to + # rotate either. Try again on the next upgrade pass. + self.mgr.log.debug('Skipping mgr/mon key rotation, no mon daemons known') + return _, mons_needing_upgrade, __, ___ = self._detect_need_upgrade(mon_daemons, target_digests, target_image) need_rotate_self = False if not mons_needing_upgrade: @@ -1408,10 +1414,6 @@ def _mark_upgrade_complete(self, target_digests: Optional[List[str]], target_ima if self.upgrade_state.health_warnings_muted: self._unmute_upgrade_related_health_warnings() - if getattr(self.upgrade_state, 'noautoscale_set', False): - self._unset_noautoscale() - self.upgrade_state.noautoscale_set = False - logger.info('Upgrade: Complete!') if self.upgrade_state.progress_id: self.mgr.remote('progress', 'complete', diff --git a/src/pybind/mgr/dashboard/frontend/CMakeLists.txt b/src/pybind/mgr/dashboard/frontend/CMakeLists.txt index bb8feb7648d1..af28daeb114f 100644 --- a/src/pybind/mgr/dashboard/frontend/CMakeLists.txt +++ b/src/pybind/mgr/dashboard/frontend/CMakeLists.txt @@ -73,9 +73,12 @@ else(WITH_SYSTEM_NPM) # ensure that the files that nodeenv unpacks from tarballs are owned by # the current user. This can be an issue due to the node tarball using # uid 1000 and running the unpack in a id-mapped namespace (container) - # that lets tar set the uid to a "bad" uid outside the namespace - COMMAND bash -c 'chown -R `id -u`:`id -g` ${mgr-dashboard-nodeenv-dir}/src' - COMMAND mkdir -p ${npm-cache-dir} + # that lets tar set the uid to a "bad" uid outside the namespace. + # nodeenv >= 1.11.0 no longer leaves the src directory behind after a + # prebuilt node install, so tolerate it being absent + COMMAND bash -c 'test ! -e ${mgr-dashboard-nodeenv-dir}/src || chown -R `id -u`:`id -g` ${mgr-dashboard-nodeenv-dir}/src' + COMMAND ${CMAKE_COMMAND} -E make_directory ${npm-cache-dir} + DEPENDS ${CMAKE_SOURCE_DIR}/src/tools/setup-virtualenv.sh WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMENT "dashboard nodeenv is being installed") if(DEFINED ENV{NPM_REGISTRY}) diff --git a/src/rgw/radosgw-admin/radosgw-admin.cc b/src/rgw/radosgw-admin/radosgw-admin.cc index e77ec3d0baf2..c52ba01dcdd8 100644 --- a/src/rgw/radosgw-admin/radosgw-admin.cc +++ b/src/rgw/radosgw-admin/radosgw-admin.cc @@ -495,10 +495,12 @@ void usage() cout << " --max-size specify max size (in B/K/M/G/T, negative value to disable)\n"; cout << " --quota-scope scope of quota (bucket, user, account)\n"; cout << "\nRate limiting options:\n"; - cout << " --max-read-ops specify max requests per minute for READ ops per RGW (GET and HEAD request methods), 0 means unlimited\n"; - cout << " --max-read-bytes specify max bytes per minute for READ ops per RGW (GET and HEAD request methods), 0 means unlimited\n"; - cout << " --max-write-ops specify max requests per minute for WRITE ops per RGW (Not GET or HEAD request methods), 0 means unlimited\n"; - cout << " --max-write-bytes specify max bytes per minute for WRITE ops per RGW (Not GET or HEAD request methods), 0 means unlimited\n"; + cout << " --max-read-ops specify max requests per accumulation interval for READ ops per RGW (GET and HEAD request methods), 0 means unlimited\n"; + cout << " --max-read-bytes specify max bytes per accumulation interval for READ ops per RGW (GET and HEAD request methods), 0 means unlimited\n"; + cout << " --max-write-ops specify max requests per accumulation interval for WRITE ops per RGW (Not GET or HEAD request methods), 0 means unlimited\n"; + cout << " --max-write-bytes specify max bytes per accumulation interval for WRITE ops per RGW (Not GET or HEAD request methods), 0 means unlimited\n"; + cout << " --max-list-ops specify max requests per accumulation interval for bucket listing requests per RGW, 0 means unlimited\n"; + cout << " --max-delete-ops specify max requests per accumulation interval for DELETE ops per RGW (DELETE request methods), 0 means unlimited\n"; cout << " --ratelimit-scope scope of rate limiting: bucket, user, anonymous\n"; cout << " anonymous can be configured only with global rate limit\n"; cout << "\nOrphans search options:\n"; @@ -1415,9 +1417,9 @@ static bool dump_string(const char *field_name, bufferlist& bl, Formatter *f) } bool set_ratelimit_info(RGWRateLimitInfo& ratelimit, OPT opt_cmd, int64_t max_read_ops, int64_t max_write_ops, - int64_t max_read_bytes, int64_t max_write_bytes, - bool have_max_read_ops, bool have_max_write_ops, - bool have_max_read_bytes, bool have_max_write_bytes) + int64_t max_list_ops, int64_t max_delete_ops, int64_t max_read_bytes, int64_t max_write_bytes, + bool have_max_read_ops, bool have_max_write_ops, bool have_max_list_ops, + bool have_max_delete_ops, bool have_max_read_bytes, bool have_max_write_bytes) { bool ratelimit_configured = true; switch (opt_cmd) { @@ -1441,6 +1443,18 @@ bool set_ratelimit_info(RGWRateLimitInfo& ratelimit, OPT opt_cmd, int64_t max_re ratelimit_configured = true; } } + if (have_max_list_ops) { + if (max_list_ops >= 0) { + ratelimit.max_list_ops = max_list_ops; + ratelimit_configured = true; + } + } + if (have_max_delete_ops) { + if (max_delete_ops >= 0) { + ratelimit.max_delete_ops = max_delete_ops; + ratelimit_configured = true; + } + } if (have_max_read_bytes) { if (max_read_bytes >= 0) { ratelimit.max_read_bytes = max_read_bytes; @@ -1525,10 +1539,10 @@ int set_bucket_quota(rgw::sal::Driver* driver, OPT opt_cmd, int set_bucket_ratelimit(rgw::sal::Driver* driver, OPT opt_cmd, const string& tenant_name, const string& bucket_name, - int64_t max_read_ops, int64_t max_write_ops, - int64_t max_read_bytes, int64_t max_write_bytes, - bool have_max_read_ops, bool have_max_write_ops, - bool have_max_read_bytes, bool have_max_write_bytes) + int64_t max_read_ops, int64_t max_write_ops, int64_t max_list_ops, + int64_t max_delete_ops, int64_t max_read_bytes, int64_t max_write_bytes, + bool have_max_read_ops, bool have_max_write_ops, bool have_max_list_ops, + bool have_max_delete_ops, bool have_max_read_bytes, bool have_max_write_bytes) { std::unique_ptr bucket; int r = driver->load_bucket(dpp(), rgw_bucket(tenant_name, bucket_name), @@ -1549,10 +1563,10 @@ int set_bucket_ratelimit(rgw::sal::Driver* driver, OPT opt_cmd, return -EIO; } } - bool ratelimit_configured = set_ratelimit_info(ratelimit_info, opt_cmd, max_read_ops, max_write_ops, - max_read_bytes, max_write_bytes, - have_max_read_ops, have_max_write_ops, - have_max_read_bytes, have_max_write_bytes); + bool ratelimit_configured = set_ratelimit_info(ratelimit_info, opt_cmd, max_read_ops, max_write_ops, max_list_ops, + max_delete_ops, max_read_bytes, max_write_bytes, + have_max_read_ops, have_max_write_ops, have_max_list_ops, + have_max_delete_ops, have_max_read_bytes, have_max_write_bytes); if (!ratelimit_configured) { ldpp_dout(dpp(), 0) << "ERROR: no rate limit values have been specified" << dendl; return -EINVAL; @@ -1570,10 +1584,10 @@ int set_bucket_ratelimit(rgw::sal::Driver* driver, OPT opt_cmd, } int set_user_ratelimit(OPT opt_cmd, std::unique_ptr& user, - int64_t max_read_ops, int64_t max_write_ops, - int64_t max_read_bytes, int64_t max_write_bytes, - bool have_max_read_ops, bool have_max_write_ops, - bool have_max_read_bytes, bool have_max_write_bytes) + int64_t max_read_ops, int64_t max_write_ops, int64_t max_list_ops, + int64_t max_delete_ops, int64_t max_read_bytes, int64_t max_write_bytes, + bool have_max_read_ops, bool have_max_write_ops, bool have_max_list_ops, + bool have_max_delete_ops, bool have_max_read_bytes, bool have_max_write_bytes) { RGWRateLimitInfo ratelimit_info; user->load_user(dpp(), null_yield); @@ -1588,10 +1602,10 @@ int set_user_ratelimit(OPT opt_cmd, std::unique_ptr& user, return -EIO; } } - bool ratelimit_configured = set_ratelimit_info(ratelimit_info, opt_cmd, max_read_ops, max_write_ops, - max_read_bytes, max_write_bytes, - have_max_read_ops, have_max_write_ops, - have_max_read_bytes, have_max_write_bytes); + bool ratelimit_configured = set_ratelimit_info(ratelimit_info, opt_cmd, max_read_ops, max_write_ops, max_list_ops, + max_delete_ops, max_read_bytes, max_write_bytes, + have_max_read_ops, have_max_write_ops, have_max_list_ops, + have_max_delete_ops, have_max_read_bytes, have_max_write_bytes); if (!ratelimit_configured) { ldpp_dout(dpp(), 0) << "ERROR: no rate limit values have been specified" << dendl; return -EINVAL; @@ -1622,7 +1636,10 @@ int show_user_ratelimit(std::unique_ptr& user, Formatter *format ldpp_dout(dpp(), 0) << "ERROR: failed to decode rate limit" << dendl; return -EIO; } + } else { + return -ENOENT; } + formatter->open_object_section("user_ratelimit"); encode_json("user_ratelimit", ratelimit_info, formatter); formatter->close_section(); @@ -3689,12 +3706,16 @@ int main(int argc, const char **argv) int64_t max_size = -1; int64_t max_read_ops = 0; int64_t max_write_ops = 0; + int64_t max_list_ops = 0; + int64_t max_delete_ops = 0; int64_t max_read_bytes = 0; int64_t max_write_bytes = 0; bool have_max_objects = false; bool have_max_size = false; bool have_max_write_ops = false; bool have_max_read_ops = false; + bool have_max_list_ops = false; + bool have_max_delete_ops = false; bool have_max_write_bytes = false; bool have_max_read_bytes = false; int include_all = false; @@ -3984,6 +4005,20 @@ int main(int argc, const char **argv) return EINVAL; } have_max_read_ops = true; + } else if (ceph_argparse_witharg(args, i, &val, "--max-list-ops", (char*)NULL)) { + max_list_ops = (int64_t)strict_strtoll(val.c_str(), 10, &err); + if (!err.empty()) { + cerr << "ERROR: failed to parse max list requests: " << err << std::endl; + return EINVAL; + } + have_max_list_ops = true; + } else if (ceph_argparse_witharg(args, i, &val, "--max-delete-ops", (char*)NULL)) { + max_delete_ops = (int64_t)strict_strtoll(val.c_str(), 10, &err); + if (!err.empty()) { + cerr << "ERROR: failed to parse max delete requests: " << err << std::endl; + return EINVAL; + } + have_max_delete_ops = true; } else if (ceph_argparse_witharg(args, i, &val, "--max-write-ops", (char*)NULL)) { max_write_ops = (int64_t)strict_strtoll(val.c_str(), 10, &err); if (!err.empty()) { @@ -4972,24 +5007,24 @@ int main(int argc, const char **argv) formatter->open_object_section("period_config"); if (ratelimit_scope == "bucket") { ratelimit_configured = set_ratelimit_info(period_config.bucket_ratelimit, opt_cmd, - max_read_ops, max_write_ops, + max_read_ops, max_write_ops, max_list_ops, max_delete_ops, max_read_bytes, max_write_bytes, - have_max_read_ops, have_max_write_ops, - have_max_read_bytes, have_max_write_bytes); + have_max_read_ops, have_max_write_ops, have_max_list_ops, + have_max_delete_ops, have_max_read_bytes, have_max_write_bytes); encode_json("bucket_ratelimit", period_config.bucket_ratelimit, formatter.get()); } else if (ratelimit_scope == "user") { ratelimit_configured = set_ratelimit_info(period_config.user_ratelimit, opt_cmd, - max_read_ops, max_write_ops, + max_read_ops, max_write_ops, max_list_ops, max_delete_ops, max_read_bytes, max_write_bytes, - have_max_read_ops, have_max_write_ops, - have_max_read_bytes, have_max_write_bytes); + have_max_read_ops, have_max_write_ops, have_max_list_ops, + have_max_delete_ops, have_max_read_bytes, have_max_write_bytes); encode_json("user_ratelimit", period_config.user_ratelimit, formatter.get()); } else if (ratelimit_scope == "anonymous") { ratelimit_configured = set_ratelimit_info(period_config.anon_ratelimit, opt_cmd, - max_read_ops, max_write_ops, + max_read_ops, max_write_ops, max_list_ops,max_delete_ops, max_read_bytes, max_write_bytes, - have_max_read_ops, have_max_write_ops, - have_max_read_bytes, have_max_write_bytes); + have_max_read_ops, have_max_write_ops, have_max_list_ops, + have_max_delete_ops, have_max_read_bytes, have_max_write_bytes); encode_json("anonymous_ratelimit", period_config.anon_ratelimit, formatter.get()); } else if (ratelimit_scope.empty() && opt_cmd == OPT::GLOBAL_RATELIMIT_GET) { // if no scope is given for GET, print both @@ -6726,7 +6761,8 @@ int main(int argc, const char **argv) OPT::ROLE_CREATE, OPT::ROLE_DELETE, OPT::ROLE_POLICY_PUT, OPT::ROLE_POLICY_DELETE, OPT::ROLE_POLICY_ATTACH, OPT::ROLE_POLICY_DETACH, - OPT::USER_POLICY_ATTACH, OPT::USER_POLICY_DETACH}; + OPT::USER_POLICY_ATTACH, OPT::USER_POLICY_DETACH, + OPT::RATELIMIT_SET, OPT::RATELIMIT_ENABLE, OPT::RATELIMIT_DISABLE}; bool print_warning_message = (non_master_ops_list.find(opt_cmd) != non_master_ops_list.end() && non_master_cmd); @@ -11340,15 +11376,15 @@ int main(int argc, const char **argv) return EINVAL; } return set_bucket_ratelimit(driver, opt_cmd, tenant, bucket_name, - max_read_ops, max_write_ops, + max_read_ops, max_write_ops, max_list_ops, max_delete_ops, max_read_bytes, max_write_bytes, - have_max_read_ops, have_max_write_ops, + have_max_read_ops, have_max_write_ops, have_max_list_ops, have_max_delete_ops, have_max_read_bytes, have_max_write_bytes); } else if (!rgw::sal::User::empty(user)) { if (ratelimit_scope == "user") { - return set_user_ratelimit(opt_cmd, user, max_read_ops, max_write_ops, + return set_user_ratelimit(opt_cmd, user, max_read_ops, max_write_ops, max_list_ops, max_delete_ops, max_read_bytes, max_write_bytes, - have_max_read_ops, have_max_write_ops, + have_max_read_ops, have_max_write_ops, have_max_list_ops, have_max_delete_ops, have_max_read_bytes, have_max_write_bytes); } else { cerr << "ERROR: invalid ratelimit scope specification. Please specify either --ratelimit-scope=bucket, or --ratelimit-scope=user" << std::endl; @@ -11371,7 +11407,11 @@ int main(int argc, const char **argv) return show_bucket_ratelimit(driver, tenant, bucket_name, formatter.get()); } else if (!rgw::sal::User::empty(user)) { if (ratelimit_scope == "user") { - return show_user_ratelimit(user, formatter.get()); + int ret = show_user_ratelimit(user, formatter.get()); + if (ret < 0) { + std::cerr << "ERROR: failed to get a ratelimit for user id: '" << user->get_id() << "', errno: " << cpp_strerror(-ret) << std::endl; + } + return ret; } else { cerr << "ERROR: invalid ratelimit scope specification. Please specify either --ratelimit-scope=bucket, or --ratelimit-scope=user" << std::endl; return EINVAL; diff --git a/src/rgw/rgw_auth.cc b/src/rgw/rgw_auth.cc index 74805fbb186b..ab7cc143a030 100644 --- a/src/rgw/rgw_auth.cc +++ b/src/rgw/rgw_auth.cc @@ -1033,6 +1033,18 @@ void rgw::auth::RemoteApplier::modify_request_state(const DoutPrefixProvider* dp // copy our identity policies into req_state s->iam_identity_policies.insert(s->iam_identity_policies.end(), policies.begin(), policies.end()); + + for (auto role : this->info.keystone_roles) { + // Keystone roles are case-insensitive. Normalize the roles to + // lowercase before placing them into the environment. + std::transform(role.begin(), role.end(), role.begin(), + [](unsigned char c){ return std::tolower(c); }); + s->env.emplace("keystone:role", std::move(role)); + } + + if (!info.keystone_user_id.empty()) { + s->env.emplace("keystone:userid", info.keystone_user_id); + } } /* rgw::auth::LocalApplier */ diff --git a/src/rgw/rgw_auth.h b/src/rgw/rgw_auth.h index b29f863d030f..10009c5d8f6e 100644 --- a/src/rgw/rgw_auth.h +++ b/src/rgw/rgw_auth.h @@ -592,6 +592,8 @@ class RemoteApplier : public IdentityApplier { const std::string access_key_id; const std::string subuser; const std::optional keystone_scope; + const std::vector keystone_roles; + const std::string keystone_user_id; public: enum class acct_privilege_t { @@ -609,7 +611,10 @@ class RemoteApplier : public IdentityApplier { const std::string access_key_id, const std::string subuser, const uint32_t acct_type=TYPE_NONE, - std::optional keystone_scope=std::nullopt) + std::optional keystone_scope=std::nullopt, + std::vector keystone_roles = {}, + const std::string keystone_user_id = {} + ) : acct_user(acct_user), acct_name(acct_name), perm_mask(perm_mask), @@ -617,7 +622,9 @@ class RemoteApplier : public IdentityApplier { acct_type(acct_type), access_key_id(access_key_id), subuser(subuser), - keystone_scope(std::move(keystone_scope)) { + keystone_scope(std::move(keystone_scope)), + keystone_roles(std::move(keystone_roles)), + keystone_user_id(keystone_user_id) { } }; diff --git a/src/rgw/rgw_auth_keystone.cc b/src/rgw/rgw_auth_keystone.cc index e0678e6fb9f5..2185efd81082 100644 --- a/src/rgw/rgw_auth_keystone.cc +++ b/src/rgw/rgw_auth_keystone.cc @@ -144,13 +144,14 @@ TokenEngine::get_creds_info(const TokenEngine::token_envelope_t& token ) const noexcept { using acct_privilege_t = rgw::auth::RemoteApplier::AuthInfo::acct_privilege_t; + std::vector role_names; /* Check whether the user has an admin status. */ acct_privilege_t level = acct_privilege_t::IS_PLAIN_ACCT; for (const auto& role : token.roles) { + role_names.push_back(role.name); if (role.is_admin && !role.is_reader) { level = acct_privilege_t::IS_ADMIN_ACCT; - break; } } @@ -169,8 +170,10 @@ TokenEngine::get_creds_info(const TokenEngine::token_envelope_t& token rgw::auth::RemoteApplier::AuthInfo::NO_ACCESS_KEY, rgw::auth::RemoteApplier::AuthInfo::NO_SUBUSER, TYPE_KEYSTONE, - std::move(keystone_scope) -}; + std::move(keystone_scope), + std::move(role_names), + token.get_user_id() + }; } static inline const std::string @@ -673,6 +676,11 @@ EC2Engine::get_creds_info(const EC2Engine::token_envelope_t& token, /* Build keystone scope info if ops logging is enabled */ auto keystone_scope = rgw::keystone::build_scope_info(cct, token); + std::vector role_names; + for (const auto& role : token.roles) { + role_names.push_back(role.name); + } + return auth_info_t { /* Suggested account name for the authenticated user. */ rgw_user(token.get_project_id()), @@ -685,7 +693,9 @@ EC2Engine::get_creds_info(const EC2Engine::token_envelope_t& token, access_key_id, rgw::auth::RemoteApplier::AuthInfo::NO_SUBUSER, TYPE_KEYSTONE, - std::move(keystone_scope) + std::move(keystone_scope), + std::move(role_names), + token.get_user_id() }; } diff --git a/src/rgw/rgw_common.cc b/src/rgw/rgw_common.cc index e4c9dda3a5cf..4599d45f3426 100644 --- a/src/rgw/rgw_common.cc +++ b/src/rgw/rgw_common.cc @@ -2829,6 +2829,8 @@ void RGWRateLimitInfo::dump(Formatter *f) const { f->dump_int("max_read_ops", max_read_ops); f->dump_int("max_write_ops", max_write_ops); + f->dump_int("max_list_ops", max_list_ops); + f->dump_int("max_delete_ops", max_delete_ops); f->dump_int("max_read_bytes", max_read_bytes); f->dump_int("max_write_bytes", max_write_bytes); f->dump_bool("enabled", enabled); diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index b213260013d9..1f3e537b003d 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -568,26 +568,40 @@ class RateLimiter; struct RGWRateLimitInfo { int64_t max_write_ops; int64_t max_read_ops; + int64_t max_list_ops; + int64_t max_delete_ops; int64_t max_write_bytes; int64_t max_read_bytes; bool enabled = false; RGWRateLimitInfo() - : max_write_ops(0), max_read_ops(0), max_write_bytes(0), max_read_bytes(0) {} + : max_write_ops(0), max_read_ops(0), max_list_ops(0), max_delete_ops(0), max_write_bytes(0), max_read_bytes(0) {} void encode(bufferlist& bl) const { - ENCODE_START(1, 1, bl); + ENCODE_START(2, 1, bl); encode(max_write_ops, bl); encode(max_read_ops, bl); + encode(max_list_ops, bl); + encode(max_delete_ops, bl); encode(max_write_bytes, bl); encode(max_read_bytes, bl); encode(enabled, bl); ENCODE_FINISH(bl); } void decode(bufferlist::const_iterator& bl) { - DECODE_START(1, bl); - decode(max_write_ops,bl); + DECODE_START(2, bl); + decode(max_write_ops, bl); decode(max_read_ops, bl); - decode(max_write_bytes,bl); + if (struct_v >= 2) { + decode(max_list_ops, bl); + } else { + max_list_ops = 0; + } + if (struct_v >= 2) { + decode(max_delete_ops, bl); + } else { + max_delete_ops = 0; + } + decode(max_write_bytes, bl); decode(max_read_bytes, bl); decode(enabled, bl); DECODE_FINISH(bl); @@ -1289,6 +1303,7 @@ struct req_state : DoutPrefixProvider { std::shared_ptr ratelimit_data; RGWRateLimitInfo user_ratelimit; RGWRateLimitInfo bucket_ratelimit; + int64_t ratelimit_retry_after{0}; std::string ratelimit_bucket_marker; std::string ratelimit_user_name; bool content_started{false}; diff --git a/src/rgw/rgw_process.cc b/src/rgw/rgw_process.cc index 3d9c8a5767a1..1ac3e1506d1a 100644 --- a/src/rgw/rgw_process.cc +++ b/src/rgw/rgw_process.cc @@ -112,6 +112,14 @@ bool rate_limit(rgw::sal::Driver* driver, req_state* s) { s->ratelimit_bucket_marker = bucketfind; const char *method = s->info.method; + bool is_sts_user = (s->auth.identity && s->auth.identity->get_identity_type() == TYPE_ROLE); + if (is_sts_user) { + ldpp_dout(s, 21) << "STS user detected: uid=" << std::quoted(s->user->get_id().to_str()) << dendl; + auto op_ret = s->user->read_attrs(s, s->yield); + if (op_ret < 0) { + ldpp_dout(s, 0) << "checking rate_limit: uid=" << std::quoted(s->user->get_id().to_str()) << " failed to read user attrs" << dendl; + } + } auto iter = s->user->get_attrs().find(RGW_ATTR_RATELIMIT); if(iter != s->user->get_attrs().end()) { try { @@ -126,12 +134,14 @@ bool rate_limit(rgw::sal::Driver* driver, req_state* s) { ldpp_dout(s, 0) << "ERROR: failed to decode rate limit" << dendl; return -EIO; } + } else { + ldpp_dout(s, 21) << "checking rate_limit: uid=" << std::quoted(s->user->get_id().to_str()) << " does not have a rate limit attribute" << dendl; } if (s->user->get_id().id == RGW_USER_ANON_ID && global_anon.enabled) { *user_ratelimit = global_anon; } - bool limit_bucket = false; - bool limit_user = s->ratelimit_data->should_rate_limit(method, s->ratelimit_user_name, s->time, user_ratelimit); + int64_t limit_bucket = 0; + int64_t limit_user = s->ratelimit_data->should_rate_limit(method, s->ratelimit_user_name, s->time, user_ratelimit, s->info.request_params); if(!rgw::sal::Bucket::empty(s->bucket.get())) { @@ -151,14 +161,18 @@ bool rate_limit(rgw::sal::Driver* driver, req_state* s) { } } if (!limit_user) { - limit_bucket = s->ratelimit_data->should_rate_limit(method, s->ratelimit_bucket_marker, s->time, bucket_ratelimit); + limit_bucket = s->ratelimit_data->should_rate_limit(method, s->ratelimit_bucket_marker, s->time, bucket_ratelimit, s->info.request_params); } } if(limit_bucket && !limit_user) { - s->ratelimit_data->giveback_tokens(method, s->ratelimit_user_name); + s->ratelimit_data->giveback_tokens(method, s->ratelimit_user_name, s->info.request_params, user_ratelimit); } s->user_ratelimit = *user_ratelimit; s->bucket_ratelimit = *bucket_ratelimit; + int64_t delay = limit_user ? limit_user : limit_bucket; + if (delay > 0) { + s->ratelimit_retry_after = delay; + } return (limit_user || limit_bucket); } diff --git a/src/rgw/rgw_ratelimit.h b/src/rgw/rgw_ratelimit.h index 224312bb0d78..04c366500089 100644 --- a/src/rgw/rgw_ratelimit.h +++ b/src/rgw/rgw_ratelimit.h @@ -5,15 +5,33 @@ #include #include "rgw_common.h" +enum class OpType { Read, Write, List, Delete }; + class RateLimiterEntry { - /* +public: + /* fixed_point_rgw_ratelimit is important to preserve the precision of the token calculation for example: a user have a limit of single op per minute, the user will consume its single token and then will send another request, 1s after it. in that case, without this method, the user will get 0 tokens although it should get 0.016 tokens. using this method it will add 16 tokens to the user, and the user will have 16 tokens, each time rgw will do comparison rgw will divide by fixed_point_rgw_ratelimit, so the user will be blocked anyway until it has enough tokens. */ static constexpr int64_t fixed_point_rgw_ratelimit = 1000; + + // Compute the seconds needed to replenish `needed` tokens when tokens + // replenish at `limit` units per `interval` seconds. All three parameters + // are in fixed-point-scaled units (multiply user-visible token counts by + // fixed_point_rgw_ratelimit before passing). The result is in seconds, + // rounded up. Returns 0 when `limit` or `needed` is non-positive, which + // callers treat as "no delay required for this dimension". + static int64_t compute_delay(int64_t limit, int64_t needed, int64_t interval) { + if (limit <= 0 || needed <= 0) { + return 0; + } + return (needed * interval + limit - 1) / limit; + } + +private: // counters are tracked in multiples of fixed_point_rgw_ratelimit struct counters { int64_t ops = 0; @@ -21,6 +39,8 @@ class RateLimiterEntry { }; counters read; counters write; + counters list; + counters del; ceph::timespan ts; bool first_run = true; std::mutex ts_lock; @@ -33,6 +53,14 @@ class RateLimiterEntry { { return write.ops / fixed_point_rgw_ratelimit; } + int64_t list_ops() const + { + return list.ops / fixed_point_rgw_ratelimit; + } + int64_t delete_ops() const + { + return del.ops / fixed_point_rgw_ratelimit; + } int64_t read_bytes() const { return read.bytes / fixed_point_rgw_ratelimit; @@ -41,30 +69,74 @@ class RateLimiterEntry { { return write.bytes / fixed_point_rgw_ratelimit; } - bool should_rate_limit_read(int64_t ops_limit, int64_t bw_limit) { - //check if tenants did not reach their bw or ops limits and that the limits are not 0 (which is unlimited) - if(((read_ops() - 1 < 0) && (ops_limit > 0)) || - (read_bytes() < 0 && bw_limit > 0)) + // Returns 0 if the request is allowed, or the estimated retry delay + // in seconds if rate-limited. compute_delay() is the sole decision + // maker: delay > 0 means rate-limited, 0 means allowed. + int64_t should_rate_limit_read(int64_t ops_limit, int64_t bw_limit) { + const int64_t interval = g_ceph_context->_conf->rgw_ratelimit_interval; + const int64_t ops_delay = compute_delay(ops_limit * fixed_point_rgw_ratelimit, + fixed_point_rgw_ratelimit - read.ops, + interval); + const int64_t bw_delay = compute_delay(bw_limit * fixed_point_rgw_ratelimit, + -read.bytes, + interval); + const int64_t delay = std::max(ops_delay, bw_delay); + if (delay > 0) { + return delay; + } + // we don't want to reduce ops' tokens if we've rejected it. + if (read_ops() > 0) { + read.ops -= fixed_point_rgw_ratelimit; + } + return 0; + } + int64_t should_rate_limit_list(int64_t ops_limit) { - return true; + const int64_t interval = g_ceph_context->_conf->rgw_ratelimit_interval; + const int64_t delay = compute_delay(ops_limit * fixed_point_rgw_ratelimit, + fixed_point_rgw_ratelimit - list.ops, + interval); + if (delay > 0) { + return delay; + } + if (list_ops() > 0) { + list.ops -= fixed_point_rgw_ratelimit; + } + return 0; } - // we don't want to reduce ops' tokens if we've rejected it. - read.ops -= fixed_point_rgw_ratelimit; - return false; + int64_t should_rate_limit_delete(int64_t ops_limit) + { + const int64_t interval = g_ceph_context->_conf->rgw_ratelimit_interval; + const int64_t delay = compute_delay(ops_limit * fixed_point_rgw_ratelimit, + fixed_point_rgw_ratelimit - del.ops, + interval); + if (delay > 0) { + return delay; + } + if (delete_ops() > 0) { + del.ops -= fixed_point_rgw_ratelimit; + } + return 0; } - bool should_rate_limit_write(int64_t ops_limit, int64_t bw_limit) + int64_t should_rate_limit_write(int64_t ops_limit, int64_t bw_limit) { - //check if tenants did not reach their bw or ops limits and that the limits are not 0 (which is unlimited) - if(((write_ops() - 1 < 0) && (ops_limit > 0)) || - (write_bytes() < 0 && bw_limit > 0)) - { - return true; + const int64_t interval = g_ceph_context->_conf->rgw_ratelimit_interval; + const int64_t ops_delay = compute_delay(ops_limit * fixed_point_rgw_ratelimit, + fixed_point_rgw_ratelimit - write.ops, + interval); + const int64_t bw_delay = compute_delay(bw_limit * fixed_point_rgw_ratelimit, + -write.bytes, + interval); + const int64_t delay = std::max(ops_delay, bw_delay); + if (delay > 0) { + return delay; } - - // we don't want to reduce ops' tokens if we've rejected it. - write.ops -= fixed_point_rgw_ratelimit; - return false; + if (write_ops() > 0) { + write.ops -= fixed_point_rgw_ratelimit; + } + return 0; } + /* The purpose of this function is to minimum time before overriding the stored timestamp This function is necessary to force the increase tokens add at least 1 token when it updates the last stored timestamp. That way the user/bucket will not lose tokens because of rounding @@ -72,7 +144,8 @@ class RateLimiterEntry { bool minimum_time_reached(ceph::timespan curr_timestamp) const { using namespace std::chrono; - constexpr auto min_duration = duration_cast(seconds(60)) / fixed_point_rgw_ratelimit; + const size_t accumulation_interval = g_ceph_context->_conf->rgw_ratelimit_interval; + const auto min_duration = duration_cast(seconds(accumulation_interval)) / fixed_point_rgw_ratelimit; const auto delta = curr_timestamp - ts; if (delta < min_duration) { @@ -91,35 +164,49 @@ class RateLimiterEntry { write.bytes = info->max_write_bytes * fixed_point; read.ops = info->max_read_ops * fixed_point; read.bytes = info->max_read_bytes * fixed_point; + list.ops = info->max_list_ops * fixed_point; + del.ops = info->max_delete_ops * fixed_point; ts = curr_timestamp; first_run = false; return; } else if(curr_timestamp > ts && minimum_time_reached(curr_timestamp)) { - const int64_t time_in_ms = std::chrono::duration_cast(curr_timestamp - ts).count() / 60.0 / std::milli::den * fixed_point; // / 60 to make it work with 1 min token bucket + const size_t accumulation_interval = g_ceph_context->_conf->rgw_ratelimit_interval; + const int64_t time_in_ms = std::chrono::duration_cast(curr_timestamp - ts).count() / static_cast(accumulation_interval) / std::milli::den * fixed_point; ts = curr_timestamp; const int64_t write_ops = info->max_write_ops * time_in_ms; const int64_t write_bw = info->max_write_bytes * time_in_ms; const int64_t read_ops = info->max_read_ops * time_in_ms; const int64_t read_bw = info->max_read_bytes * time_in_ms; + const int64_t list_ops = info->max_list_ops * time_in_ms; + const int64_t delete_ops = info->max_delete_ops * time_in_ms; read.ops = std::min(info->max_read_ops * fixed_point, read_ops + read.ops); read.bytes = std::min(info->max_read_bytes * fixed_point, read_bw + read.bytes); write.ops = std::min(info->max_write_ops * fixed_point, write_ops + write.ops); write.bytes = std::min(info->max_write_bytes * fixed_point, write_bw + write.bytes); + list.ops = std::min(info->max_list_ops * fixed_point, list_ops + list.ops); + del.ops = std::min(info->max_delete_ops * fixed_point, delete_ops + del.ops); } } public: - bool should_rate_limit(bool is_read, const RGWRateLimitInfo* ratelimit_info, ceph::timespan curr_timestamp) + // Returns 0 if allowed, or the retry delay in seconds if rate-limited. + int64_t should_rate_limit(OpType op_type, const RGWRateLimitInfo* ratelimit_info, ceph::timespan curr_timestamp) { std::unique_lock lock(ts_lock); increase_tokens(curr_timestamp, ratelimit_info); - if (is_read) - { - return should_rate_limit_read(ratelimit_info->max_read_ops, ratelimit_info->max_read_bytes); + switch (op_type) { + case OpType::Read: + return should_rate_limit_read(ratelimit_info->max_read_ops, ratelimit_info->max_read_bytes); + case OpType::Write: + return should_rate_limit_write(ratelimit_info->max_write_ops, ratelimit_info->max_write_bytes); + case OpType::List: + return should_rate_limit_list(ratelimit_info->max_list_ops); + case OpType::Delete: + return should_rate_limit_delete(ratelimit_info->max_delete_ops); } - return should_rate_limit_write(ratelimit_info->max_write_ops, ratelimit_info->max_write_bytes); + return 0; } void decrease_bytes(bool is_read, int64_t amount, const RGWRateLimitInfo* info) { std::unique_lock lock(ts_lock); @@ -131,19 +218,27 @@ class RateLimiterEntry { write.bytes = std::max(write.bytes - amount * fixed_point_rgw_ratelimit,info->max_write_bytes * fixed_point_rgw_ratelimit * -2); } } - void giveback_tokens(bool is_read) + void giveback_tokens(OpType op_type) { std::unique_lock lock(ts_lock); - if (is_read) - { - read.ops += fixed_point_rgw_ratelimit; - } else { - write.ops += fixed_point_rgw_ratelimit; + switch (op_type) { + case OpType::Read: + read.ops += fixed_point_rgw_ratelimit; + break; + case OpType::Write: + write.ops += fixed_point_rgw_ratelimit; + break; + case OpType::List: + list.ops += fixed_point_rgw_ratelimit; + break; + case OpType::Delete: + del.ops += fixed_point_rgw_ratelimit; + break; } } }; -class RateLimiter { +class RateLimiter : public DoutPrefix { static constexpr size_t map_size = 2000000; // will create it with the closest upper prime number std::shared_mutex insert_lock; @@ -151,12 +246,31 @@ class RateLimiter { std::condition_variable& cv; typedef std::unordered_map hash_map; hash_map ratelimit_entries{map_size}; - static bool is_read_op(const std::string_view method) { + + static inline constexpr std::string_view RESOURCE_PATTERN_LIST_TYPE = "list-type="; + static inline constexpr std::string_view RESOURCE_PATTERN_PREFIX = "prefix="; + static inline constexpr std::string_view RESOURCE_PATTERN_DELIMITER = "delimiter="; + + + static OpType op_type(const std::string_view method, const std::string_view resource, const RGWRateLimitInfo* ratelimit_info) { + const bool ratelimit_list = ratelimit_info && (ratelimit_info->max_list_ops > 0); + const bool ratelimit_delete = ratelimit_info && (ratelimit_info->max_delete_ops > 0); + + auto contains_any = [](std::string_view s, auto&&... patterns) { + return ((s.find(patterns) != std::string::npos) || ...); + }; + if (ratelimit_list && method == "GET" && + !resource.empty() && contains_any(resource, RESOURCE_PATTERN_LIST_TYPE, RESOURCE_PATTERN_PREFIX, RESOURCE_PATTERN_DELIMITER)) { + return OpType::List; + } if (method == "GET" || method == "HEAD") { - return true; + return OpType::Read; + } + if (ratelimit_delete && method == "DELETE") { + return OpType::Delete; } - return false; + return OpType::Write; } // find or create an entry, and return its iterator @@ -187,41 +301,48 @@ class RateLimiter { RateLimiter(RateLimiter&&) = delete; RateLimiter& operator =(RateLimiter&&) = delete; RateLimiter() = delete; - RateLimiter(std::atomic_bool& replacing, std::condition_variable& cv) - : replacing(replacing), cv(cv) + RateLimiter(CephContext* cct, std::atomic_bool& replacing, std::condition_variable& cv) + : DoutPrefix(cct, ceph_subsys_rgw, "rate limiter: "), replacing(replacing), cv(cv) { // prevents rehash, so no iterators invalidation ratelimit_entries.max_load_factor(1000); }; - bool should_rate_limit(const char *method, const std::string& key, ceph::coarse_real_time curr_timestamp, const RGWRateLimitInfo* ratelimit_info) { + // Returns 0 if allowed, or the retry delay in seconds if rate-limited. + int64_t should_rate_limit(const char *method, const std::string& key, ceph::coarse_real_time curr_timestamp, const RGWRateLimitInfo* ratelimit_info, const std::string& resource) { + ldpp_dout(this, 21) << "checking should_rate_limit: key=" << std::quoted(key) << " enabled=" << ratelimit_info->enabled << dendl; if (key.empty() || key.length() == 1 || !ratelimit_info->enabled) { - return false; + return 0; } - bool is_read = is_read_op(method); + OpType type = op_type(method, resource, ratelimit_info); auto& it = find_or_create(key); auto curr_ts = curr_timestamp.time_since_epoch(); - return it.should_rate_limit(is_read ,ratelimit_info, curr_ts); + return it.should_rate_limit(type, ratelimit_info, curr_ts); } - void giveback_tokens(const char *method, const std::string& key) + void giveback_tokens(const char *method, const std::string& key, const std::string& resource, const RGWRateLimitInfo* ratelimit_info) { - bool is_read = is_read_op(method); + OpType type = op_type(method, resource, ratelimit_info); auto& it = find_or_create(key); - it.giveback_tokens(is_read); + it.giveback_tokens(type); } void decrease_bytes(const char *method, const std::string& key, const int64_t amount, const RGWRateLimitInfo* info) { if (key.empty() || key.length() == 1 || !info->enabled) { return; } - bool is_read = is_read_op(method); - if ((is_read && !info->max_read_bytes) || (!is_read && !info->max_write_bytes)) + OpType type = op_type(method, "", info); + // Only read and write ops affect bytes + if ((type == OpType::Read && !info->max_read_bytes) || (type == OpType::Write && !info->max_write_bytes)) { return; } auto& it = find_or_create(key); - it.decrease_bytes(is_read, amount, info); + if (type == OpType::Read) + it.decrease_bytes(true, amount, info); + else if (type == OpType::Write) + it.decrease_bytes(false, amount, info); + // OpType::List does not affect bytes } void clear() { ratelimit_entries.clear(); @@ -271,8 +392,8 @@ class ActiveRateLimiter : public DoutPrefix { ActiveRateLimiter(CephContext* cct) : DoutPrefix(cct, ceph_subsys_rgw, "rate limiter: ") { - ratelimit[0] = std::make_shared(replacing, cv); - ratelimit[1] = std::make_shared(replacing, cv); + ratelimit[0] = std::make_shared(cct, replacing, cv); + ratelimit[1] = std::make_shared(cct, replacing, cv); } ~ActiveRateLimiter() { ldpp_dout(this, 20) << "stopping ratelimit_gc thread" << dendl; diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc index 4152feee951d..813a1ff12581 100644 --- a/src/rgw/rgw_rest.cc +++ b/src/rgw/rgw_rest.cc @@ -717,6 +717,10 @@ void abort_early(req_state *s, RGWOp* op, int err_no, } dump_errno(s); + if (err_no == -ERR_RATE_LIMITED && s->ratelimit_retry_after > 0) { + dump_header(s, "Retry-After", + static_cast(s->ratelimit_retry_after)); + } dump_bucket_from_state(s); if (err_no == -ERR_PERMANENT_REDIRECT || err_no == -ERR_WEBSITE_REDIRECT) { string dest_uri; diff --git a/src/rgw/rgw_rest_ratelimit.cc b/src/rgw/rgw_rest_ratelimit.cc index 128e44d9a5ae..32011212f48d 100644 --- a/src/rgw/rgw_rest_ratelimit.cc +++ b/src/rgw/rgw_rest_ratelimit.cc @@ -1,6 +1,9 @@ // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- // vim: ts=8 sw=2 smarttab ft=cpp #include "rgw_rest_ratelimit.h" +#include "rgw_process_env.h" +#include "rgw_op.h" + class RGWOp_Ratelimit_Info : public RGWRESTOp { int check_caps(const RGWUserCaps& caps) override { return caps.check_cap("ratelimit", RGW_CAP_READ); @@ -130,12 +133,14 @@ class RGWOp_Ratelimit_Set : public RGWRESTOp { void set_ratelimit_info(bool have_max_read_ops, int64_t max_read_ops, bool have_max_write_ops, int64_t max_write_ops, bool have_max_read_bytes, int64_t max_read_bytes, bool have_max_write_bytes, int64_t max_write_bytes, + bool have_max_list_ops, int64_t max_list_ops, bool have_max_delete_ops, int64_t max_delete_ops, bool have_enabled, bool enabled, bool& ratelimit_configured, RGWRateLimitInfo& ratelimit_info); }; void RGWOp_Ratelimit_Set::set_ratelimit_info(bool have_max_read_ops, int64_t max_read_ops, bool have_max_write_ops, int64_t max_write_ops, bool have_max_read_bytes, int64_t max_read_bytes, bool have_max_write_bytes, int64_t max_write_bytes, + bool have_max_list_ops, int64_t max_list_ops, bool have_max_delete_ops, int64_t max_delete_ops, bool have_enabled, bool enabled, bool& ratelimit_configured, RGWRateLimitInfo& ratelimit_info) { if (have_max_read_ops) { @@ -162,6 +167,18 @@ class RGWOp_Ratelimit_Set : public RGWRESTOp { ratelimit_configured = true; } } + if (have_max_list_ops) { + if (max_list_ops >= 0) { + ratelimit_info.max_list_ops = max_list_ops; + ratelimit_configured = true; + } + } + if (have_max_delete_ops) { + if (max_delete_ops >= 0) { + ratelimit_info.max_delete_ops = max_delete_ops; + ratelimit_configured = true; + } + } if (have_enabled) { ratelimit_info.enabled = enabled; ratelimit_configured = true; @@ -194,6 +211,10 @@ void RGWOp_Ratelimit_Set::execute(optional_yield y) bool have_max_read_bytes = false; int64_t max_write_bytes = 0; bool have_max_write_bytes = false; + int64_t max_list_ops = 0; + bool have_max_list_ops = false; + int64_t max_delete_ops = 0; + bool have_max_delete_ops = false; RESTArgs::get_string(s, "uid", uid_str, &uid_str); RESTArgs::get_string(s, "ratelimit-scope", ratelimit_scope, &ratelimit_scope); RESTArgs::get_string(s, "bucket", bucket_name, &bucket_name); @@ -203,6 +224,8 @@ void RGWOp_Ratelimit_Set::execute(optional_yield y) op_ret |= RESTArgs::get_int64(s, "max-write-ops", 0, &max_write_ops, &have_max_write_ops); op_ret |= RESTArgs::get_int64(s, "max-read-bytes", 0, &max_read_bytes, &have_max_read_bytes); op_ret |= RESTArgs::get_int64(s, "max-write-bytes", 0, &max_write_bytes, &have_max_write_bytes); + op_ret |= RESTArgs::get_int64(s, "max-list-ops", 0, &max_list_ops, &have_max_list_ops); + op_ret |= RESTArgs::get_int64(s, "max-delete-ops", 0, &max_delete_ops, &have_max_delete_ops); if (op_ret) { ldpp_dout(this, 0) << "one of the maximum arguments could not be parsed" << dendl; return; @@ -227,8 +250,18 @@ void RGWOp_Ratelimit_Set::execute(optional_yield y) } } RESTArgs::get_bool(s, "global", false, &global, nullptr); + + // forward to master zonegroup + op_ret = rgw_forward_request_to_master(this, *s->penv.site, s->user->get_id(), + nullptr, nullptr, s->info, s->err, y); + if (op_ret < 0) { + ldpp_dout(this, 0) << "ERROR: forward_request_to_master returned ret=" << op_ret << dendl; + return; + } + set_ratelimit_info(have_max_read_ops, max_read_ops, have_max_write_ops, max_write_ops, have_max_read_bytes, max_read_bytes, have_max_write_bytes, max_write_bytes, + have_max_list_ops, max_list_ops, have_max_delete_ops, max_delete_ops, have_enabled, enabled, ratelimit_configured, ratelimit_info); if (op_ret) { return; @@ -262,6 +295,7 @@ void RGWOp_Ratelimit_Set::execute(optional_yield y) } set_ratelimit_info(have_max_read_ops, max_read_ops, have_max_write_ops, max_write_ops, have_max_read_bytes, max_read_bytes, have_max_write_bytes, max_write_bytes, + have_max_list_ops, max_list_ops, have_max_delete_ops, max_delete_ops, have_enabled, enabled, ratelimit_configured, ratelimit_info); bufferlist bl; ratelimit_info.encode(bl); @@ -295,6 +329,7 @@ void RGWOp_Ratelimit_Set::execute(optional_yield y) bufferlist bl; set_ratelimit_info(have_max_read_ops, max_read_ops, have_max_write_ops, max_write_ops, have_max_read_bytes, max_read_bytes, have_max_write_bytes, max_write_bytes, + have_max_list_ops, max_list_ops, have_max_delete_ops, max_delete_ops, have_enabled, enabled, ratelimit_configured, ratelimit_info); ratelimit_info.encode(bl); rgw::sal::Attrs attr; @@ -314,6 +349,7 @@ void RGWOp_Ratelimit_Set::execute(optional_yield y) ratelimit_info = period_config.bucket_ratelimit; set_ratelimit_info(have_max_read_ops, max_read_ops, have_max_write_ops, max_write_ops, have_max_read_bytes, max_read_bytes, have_max_write_bytes, max_write_bytes, + have_max_list_ops, max_list_ops, have_max_delete_ops, max_delete_ops, have_enabled, enabled, ratelimit_configured, ratelimit_info); period_config.bucket_ratelimit = ratelimit_info; op_ret = period_config.write(s, static_cast(driver)->svc()->sysobj, realm_id, y); @@ -323,6 +359,7 @@ void RGWOp_Ratelimit_Set::execute(optional_yield y) ratelimit_info = period_config.anon_ratelimit; set_ratelimit_info(have_max_read_ops, max_read_ops, have_max_write_ops, max_write_ops, have_max_read_bytes, max_read_bytes, have_max_write_bytes, max_write_bytes, + have_max_list_ops, max_list_ops, have_max_delete_ops, max_delete_ops, have_enabled, enabled, ratelimit_configured, ratelimit_info); period_config.anon_ratelimit = ratelimit_info; op_ret = period_config.write(s, static_cast(driver)->svc()->sysobj, realm_id, y); @@ -332,6 +369,7 @@ void RGWOp_Ratelimit_Set::execute(optional_yield y) ratelimit_info = period_config.user_ratelimit; set_ratelimit_info(have_max_read_ops, max_read_ops, have_max_write_ops, max_write_ops, have_max_read_bytes, max_read_bytes, have_max_write_bytes, max_write_bytes, + have_max_list_ops, max_list_ops, have_max_delete_ops, max_delete_ops, have_enabled, enabled, ratelimit_configured, ratelimit_info); period_config.user_ratelimit = ratelimit_info; op_ret = period_config.write(s, static_cast(driver)->svc()->sysobj, realm_id, y); diff --git a/src/test/cli/ceph-authtool/help.t b/src/test/cli/ceph-authtool/help.t index 68f4a9699587..9d8b0cf67bc1 100644 --- a/src/test/cli/ceph-authtool/help.t +++ b/src/test/cli/ceph-authtool/help.t @@ -11,6 +11,7 @@ existing keyringfile -g, --gen-key will generate a new secret key for the specified entityname + -t, --key-type KEY_TYPE set the key-type --gen-print-key will generate a new secret key without set it to the keyringfile, prints the secret to stdout --import-keyring FILE will import the content of a given keyring diff --git a/src/test/cli/monmaptool/add-exists.t b/src/test/cli/monmaptool/add-exists.t index d223f7bec8d6..4220a4b75855 100644 --- a/src/test/cli/monmaptool/add-exists.t +++ b/src/test/cli/monmaptool/add-exists.t @@ -24,6 +24,10 @@ min_mon_release 20 (tentacle) election_strategy: 1 0: v1:2.3.4.5:6789/0 mon.foo + auth_epoch 0 + auth_service_cipher aes256k + auth_allowed_ciphers aes256k + auth_preferred_cipher aes256k $ NEW_FSID="$(monmaptool --print mymonmap|grep ^fsid)" $ [ "$ORIG_FSID" = "$NEW_FSID" ] diff --git a/src/test/cli/monmaptool/add-many.t b/src/test/cli/monmaptool/add-many.t index 2f61888c9e9d..1d5de755846c 100644 --- a/src/test/cli/monmaptool/add-many.t +++ b/src/test/cli/monmaptool/add-many.t @@ -30,6 +30,10 @@ 1: [v2:172.21.15.68:6791/0,v1:172.21.15.68:6792/0] mon.fiz 2: v2:3.4.5.6:7890/0 mon.bar 3: v2:4.5.6.7:8901/0 mon.baz + auth_epoch 0 + auth_service_cipher aes256k + auth_allowed_ciphers aes256k + auth_preferred_cipher aes256k $ NEW_FSID="$(monmaptool --print mymonmap|grep ^fsid)" $ [ "$ORIG_FSID" = "$NEW_FSID" ] diff --git a/src/test/cli/monmaptool/clobber.t b/src/test/cli/monmaptool/clobber.t index 963e1ebeda1c..779ac895166c 100644 --- a/src/test/cli/monmaptool/clobber.t +++ b/src/test/cli/monmaptool/clobber.t @@ -21,6 +21,10 @@ min_mon_release 20 (tentacle) election_strategy: 1 0: v1:2.3.4.5:6789/0 mon.foo + auth_epoch 0 + auth_service_cipher aes256k + auth_allowed_ciphers aes256k + auth_preferred_cipher aes256k $ NEW_FSID="$(monmaptool --print mymonmap|grep ^fsid)" $ [ "$ORIG_FSID" = "$NEW_FSID" ] @@ -42,3 +46,7 @@ created \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+.\d\d\d\d (re) min_mon_release 20 (tentacle) election_strategy: 1 + auth_epoch 0 + auth_service_cipher aes256k + auth_allowed_ciphers aes256k + auth_preferred_cipher aes256k diff --git a/src/test/cli/monmaptool/create-print.t b/src/test/cli/monmaptool/create-print.t index bf99d2589930..542cab69eb37 100644 --- a/src/test/cli/monmaptool/create-print.t +++ b/src/test/cli/monmaptool/create-print.t @@ -12,6 +12,10 @@ created \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+.\d\d\d\d (re) min_mon_release 20 (tentacle) election_strategy: 1 + auth_epoch 0 + auth_service_cipher aes256k + auth_allowed_ciphers aes256k + auth_preferred_cipher aes256k $ monmaptool --print -- mymonmap monmaptool: monmap file mymonmap @@ -21,3 +25,7 @@ created \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+.\d\d\d\d (re) min_mon_release 20 (tentacle) election_strategy: 1 + auth_epoch 0 + auth_service_cipher aes256k + auth_allowed_ciphers aes256k + auth_preferred_cipher aes256k diff --git a/src/test/cli/monmaptool/create-with-add.t b/src/test/cli/monmaptool/create-with-add.t index 9aaaa7e6c8d3..8350c1c876f0 100644 --- a/src/test/cli/monmaptool/create-with-add.t +++ b/src/test/cli/monmaptool/create-with-add.t @@ -13,3 +13,7 @@ min_mon_release 20 (tentacle) election_strategy: 1 0: v1:2.3.4.5:6789/0 mon.foo + auth_epoch 0 + auth_service_cipher aes256k + auth_allowed_ciphers aes256k + auth_preferred_cipher aes256k diff --git a/src/test/cli/monmaptool/feature-set-unset-list.t b/src/test/cli/monmaptool/feature-set-unset-list.t index 65841b7600ef..ca303ef39dc0 100644 --- a/src/test/cli/monmaptool/feature-set-unset-list.t +++ b/src/test/cli/monmaptool/feature-set-unset-list.t @@ -12,21 +12,21 @@ required: [none] AVAILABLE FEATURES: - supported: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)] - persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)] + supported: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),cephx_auth_aes256k(2147483648),nvmeof_beacon_diff(4294967296)] + persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),cephx_auth_aes256k(2147483648),nvmeof_beacon_diff(4294967296)] MONMAP FEATURES: persistent: [none] optional: [none] required: [none] AVAILABLE FEATURES: - supported: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)] - persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)] + supported: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),cephx_auth_aes256k(2147483648),nvmeof_beacon_diff(4294967296)] + persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),cephx_auth_aes256k(2147483648),nvmeof_beacon_diff(4294967296)] monmap:persistent:[none] monmap:optional:[none] monmap:required:[none] - available:supported:[kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)] - available:persistent:[kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)] + available:supported:[kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),cephx_auth_aes256k(2147483648),nvmeof_beacon_diff(4294967296)] + available:persistent:[kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),cephx_auth_aes256k(2147483648),nvmeof_beacon_diff(4294967296)] $ monmaptool --feature-set foo /tmp/test.monmap.1234 unknown features name 'foo' or unable to parse value: Expected option value to be integer, got 'foo' @@ -45,8 +45,8 @@ required: [kraken(1),octopus(32),unknown(4096)] AVAILABLE FEATURES: - supported: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)] - persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)] + supported: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),cephx_auth_aes256k(2147483648),nvmeof_beacon_diff(4294967296)] + persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),cephx_auth_aes256k(2147483648),nvmeof_beacon_diff(4294967296)] $ monmaptool --feature-unset 32 --optional --feature-list /tmp/test.monmap.1234 monmaptool: monmap file /tmp/test.monmap.1234 @@ -56,8 +56,8 @@ required: [kraken(1),octopus(32),unknown(4096)] AVAILABLE FEATURES: - supported: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)] - persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)] + supported: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),cephx_auth_aes256k(2147483648),nvmeof_beacon_diff(4294967296)] + persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),cephx_auth_aes256k(2147483648),nvmeof_beacon_diff(4294967296)] monmaptool: writing epoch 0 to /tmp/test.monmap.1234 (1 monitors) $ monmaptool --feature-unset 32 --persistent --feature-unset 4096 --optional --feature-list /tmp/test.monmap.1234 @@ -68,8 +68,8 @@ required: [kraken(1)] AVAILABLE FEATURES: - supported: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)] - persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)] + supported: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),cephx_auth_aes256k(2147483648),nvmeof_beacon_diff(4294967296)] + persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),cephx_auth_aes256k(2147483648),nvmeof_beacon_diff(4294967296)] monmaptool: writing epoch 0 to /tmp/test.monmap.1234 (1 monitors) $ monmaptool --feature-unset kraken --feature-list /tmp/test.monmap.1234 @@ -80,8 +80,8 @@ required: [none] AVAILABLE FEATURES: - supported: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)] - persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)] + supported: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),cephx_auth_aes256k(2147483648),nvmeof_beacon_diff(4294967296)] + persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),cephx_auth_aes256k(2147483648),nvmeof_beacon_diff(4294967296)] monmaptool: writing epoch 0 to /tmp/test.monmap.1234 (1 monitors) $ rm /tmp/test.monmap.1234 diff --git a/src/test/cli/monmaptool/help.t b/src/test/cli/monmaptool/help.t index f12c1a07d003..e8c2223fe1f8 100644 --- a/src/test/cli/monmaptool/help.t +++ b/src/test/cli/monmaptool/help.t @@ -8,4 +8,7 @@ [--feature-set [--optional|--persistent]] [--feature-unset [--optional|--persistent]] [--set-min-mon-release ] + [--auth-service-cipher ] + [--auth-allowed-ciphers ] + [--auth-preferred-cipher ] diff --git a/src/test/cli/monmaptool/rm-nonexistent.t b/src/test/cli/monmaptool/rm-nonexistent.t index c6709f934d0b..dfb56a2a8e64 100644 --- a/src/test/cli/monmaptool/rm-nonexistent.t +++ b/src/test/cli/monmaptool/rm-nonexistent.t @@ -22,6 +22,10 @@ min_mon_release 20 (tentacle) election_strategy: 1 0: v1:2.3.4.5:6789/0 mon.foo + auth_epoch 0 + auth_service_cipher aes256k + auth_allowed_ciphers aes256k + auth_preferred_cipher aes256k $ NEW_FSID="$(monmaptool --print mymonmap|grep ^fsid)" $ [ "$ORIG_FSID" = "$NEW_FSID" ] diff --git a/src/test/cli/monmaptool/rm.t b/src/test/cli/monmaptool/rm.t index b84d15ea23f1..6194a86ea41c 100644 --- a/src/test/cli/monmaptool/rm.t +++ b/src/test/cli/monmaptool/rm.t @@ -19,6 +19,10 @@ created \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+.\d\d\d\d (re) min_mon_release 20 (tentacle) election_strategy: 1 + auth_epoch 0 + auth_service_cipher aes256k + auth_allowed_ciphers aes256k + auth_preferred_cipher aes256k $ NEW_FSID="$(monmaptool --print mymonmap|grep ^fsid)" $ [ "$ORIG_FSID" = "$NEW_FSID" ] diff --git a/src/test/cli/radosgw-admin/help.t b/src/test/cli/radosgw-admin/help.t index 5fcfb056e4e2..3599cbd72af1 100644 --- a/src/test/cli/radosgw-admin/help.t +++ b/src/test/cli/radosgw-admin/help.t @@ -358,10 +358,12 @@ --quota-scope scope of quota (bucket, user, account) Rate limiting options: - --max-read-ops specify max requests per minute for READ ops per RGW (GET and HEAD request methods), 0 means unlimited - --max-read-bytes specify max bytes per minute for READ ops per RGW (GET and HEAD request methods), 0 means unlimited - --max-write-ops specify max requests per minute for WRITE ops per RGW (Not GET or HEAD request methods), 0 means unlimited - --max-write-bytes specify max bytes per minute for WRITE ops per RGW (Not GET or HEAD request methods), 0 means unlimited + --max-read-ops specify max requests per accumulation interval for READ ops per RGW (GET and HEAD request methods), 0 means unlimited + --max-read-bytes specify max bytes per accumulation interval for READ ops per RGW (GET and HEAD request methods), 0 means unlimited + --max-write-ops specify max requests per accumulation interval for WRITE ops per RGW (Not GET or HEAD request methods), 0 means unlimited + --max-write-bytes specify max bytes per accumulation interval for WRITE ops per RGW (Not GET or HEAD request methods), 0 means unlimited + --max-list-ops specify max requests per accumulation interval for bucket listing requests per RGW, 0 means unlimited + --max-delete-ops specify max requests per accumulation interval for DELETE ops per RGW (DELETE request methods), 0 means unlimited --ratelimit-scope scope of rate limiting: bucket, user, anonymous anonymous can be configured only with global rate limit diff --git a/src/test/osd/TestOSDMap.cc b/src/test/osd/TestOSDMap.cc index 6d480c9c9a77..878c1e945d9f 100644 --- a/src/test/osd/TestOSDMap.cc +++ b/src/test/osd/TestOSDMap.cc @@ -10,6 +10,7 @@ #include "common/common_init.h" #include "common/ceph_argparse.h" #include "common/ceph_json.h" +#include "include/stringify.h" #include #include diff --git a/src/test/rgw/bench_rgw_ratelimit.cc b/src/test/rgw/bench_rgw_ratelimit.cc index 529d8f739fd9..21ed60f07b55 100644 --- a/src/test/rgw/bench_rgw_ratelimit.cc +++ b/src/test/rgw/bench_rgw_ratelimit.cc @@ -91,7 +91,7 @@ bool simulate_request(client_info& it, const RGWRateLimitInfo& info, std::shared int rw = 0; // will always use PUT method as there is no different std::string methodop = method[rw]; auto dout = DoutPrefix(g_ceph_context, ceph_subsys_rgw, "rate limiter: "); - bool to_fail = ratelimit->should_rate_limit(methodop.c_str(), it.tenant, time, &info); + bool to_fail = ratelimit->should_rate_limit(methodop.c_str(), it.tenant, time, &info, ""); if(to_fail) { it.rejected++; diff --git a/src/test/rgw/bench_rgw_ratelimit_gc.cc b/src/test/rgw/bench_rgw_ratelimit_gc.cc index ae422e1dada2..3b837b98f74e 100644 --- a/src/test/rgw/bench_rgw_ratelimit_gc.cc +++ b/src/test/rgw/bench_rgw_ratelimit_gc.cc @@ -46,7 +46,7 @@ int main(int argc, char **argv) { std::string tenant = "uuser" + std::to_string(i); auto time = ceph::coarse_real_clock::now(); - ratelimit->get_active()->should_rate_limit("PUT", tenant, time, &info); + ratelimit->get_active()->should_rate_limit("PUT", tenant, time, &info, ""); } } diff --git a/src/test/rgw/test_rgw_iam_policy.cc b/src/test/rgw/test_rgw_iam_policy.cc index b7c9eb229d4c..ec6228df7c47 100644 --- a/src/test/rgw/test_rgw_iam_policy.cc +++ b/src/test/rgw/test_rgw_iam_policy.cc @@ -1779,3 +1779,130 @@ TEST_F(ConditionTest, Null) EXPECT_TRUE(notNull.eval({{key, "admin/config.txt"}})); } } + +TEST_F(ConditionTest, KeyStoneRoleStringEquals) +{ + std::string key = "keystone:role"; + + Condition cond{TokenID::StringEquals, key.data(), key.size(), false}; + cond.vals.push_back("testrole"); + + // No roles + EXPECT_FALSE(cond.eval({})); + + // Single matching role + EXPECT_TRUE(cond.eval({{key, "testrole"}})); + + // Single non-matching role + EXPECT_FALSE(cond.eval({{key, "member"}})); + + //Multiple roles in env,one matches + Environment multi_env; + multi_env.emplace(key, "member"); + multi_env.emplace(key, "testrole"); + multi_env.emplace(key, "reader"); + EXPECT_TRUE(cond.eval(multi_env)); + + //Multiple roles in env, no match + Environment no_match_env; + no_match_env.emplace(key, "member"); + no_match_env.emplace(key, "reader"); + EXPECT_FALSE(cond.eval(no_match_env)); + + // Multiple identical roles (redundancy check) + Environment duplicate_env; + duplicate_env.emplace(key, "testrole"); + duplicate_env.emplace(key, "testrole"); + EXPECT_TRUE(cond.eval(duplicate_env)); +} + +TEST_F(ConditionTest, KeyStoneRoleNotStringEquals) +{ + std::string key = "keystone:role"; + + Condition cond{TokenID::StringNotEquals, key.data(), key.size(), false}; + cond.vals.push_back("admin"); + + // No roles + EXPECT_FALSE(cond.eval({})); + + // Role matches + EXPECT_FALSE(cond.eval({{key, "admin"}})); + + // Role doesn't match + EXPECT_TRUE(cond.eval({{key, "member"}})); + + // Multiple roles in env, one matches -> false + Environment multi_env; + multi_env.emplace(key, "member"); + multi_env.emplace(key, "admin"); + EXPECT_FALSE(multi_env.count(key) == 0); + EXPECT_FALSE(cond.eval(multi_env)); + + // Multiple roles, none match -> true + Environment no_match_env; + no_match_env.emplace(key, "member"); + no_match_env.emplace(key, "reader"); + EXPECT_TRUE(cond.eval(no_match_env)); +} + +TEST_F(ConditionTest, KeyStoneRolePolicyParsing) +{ + string keystone_role_policy = R"( + { + "Version": "2012-10-17", + "Statement": [{ + "Effect" : "Allow", + "Principal": "*", + "Action": "s3:GetObject", + "Resource": [ + "arn:aws:s3:::example_bucket/*" + ], + "Condition": { + "StringEquals": { + "keystone:role": "testrole" + } + } + }] + } + )"; + + string tenant = "arbitrary_tenant"; + boost::optional p; + + ASSERT_NO_THROW( + p = Policy(cct.get(), &tenant, keystone_role_policy, true)); + ASSERT_TRUE(p); + EXPECT_EQ(p->statements.size(), 1U); + EXPECT_EQ(p->statements[0].conditions.size(), 1U); + EXPECT_EQ(p->statements[0].conditions[0].key, "keystone:role"); + EXPECT_EQ(p->statements[0].conditions[0].vals.size(), 1U); + EXPECT_EQ(p->statements[0].conditions[0].vals[0], "testrole"); + + // Eval with matching role in environment + Environment match_env; + match_env.emplace("keystone:role", "testrole"); + EXPECT_TRUE(p->statements[0].conditions[0].eval(match_env)); + + // Eval with non-matching role + Environment nomatch_env; + nomatch_env.emplace("keystone:role", "member"); + EXPECT_FALSE(p->statements[0].conditions[0].eval(nomatch_env)); + + // Eval with multiple roles, one matching + Environment multi_env; + multi_env.emplace("keystone:role", "member"); + multi_env.emplace("keystone:role", "testrole"); + EXPECT_TRUE(p->statements[0].conditions[0].eval(multi_env)); +} + +TEST_F(ConditionTest, KeystoneUserIdStringEquals) +{ + const std::string key = "keystone:userid"; + Condition cond{TokenID::StringEquals, key.data(), key.size(), false}; + cond.vals.push_back("user-123"); + + EXPECT_FALSE(cond.eval({})); + EXPECT_TRUE(cond.eval({{key, "user-123"}})); + EXPECT_FALSE(cond.eval({{key, "user-456"}})); +} \ No newline at end of file diff --git a/src/test/rgw/test_rgw_ratelimit.cc b/src/test/rgw/test_rgw_ratelimit.cc index 01be4df48a02..3c1eeff983be 100644 --- a/src/test/rgw/test_rgw_ratelimit.cc +++ b/src/test/rgw/test_rgw_ratelimit.cc @@ -7,171 +7,204 @@ using namespace std::chrono_literals; + +TEST(RGWRateLimitEntry, compute_delay) +{ + // Disabled limit returns 0 regardless of the deficit + EXPECT_EQ(0, RateLimiterEntry::compute_delay(0, 0, 1)); + EXPECT_EQ(0, RateLimiterEntry::compute_delay(0, 1, 1)); + EXPECT_EQ(0, RateLimiterEntry::compute_delay(0, 1000, 1)); + EXPECT_EQ(0, RateLimiterEntry::compute_delay(0, 1001, 1)); + EXPECT_EQ(0, RateLimiterEntry::compute_delay(-1, 1000, 1)); + + // Zero or negative deficit returns 0 (caller treats as "no delay needed") + EXPECT_EQ(0, RateLimiterEntry::compute_delay(1000, 0, 1)); + EXPECT_EQ(0, RateLimiterEntry::compute_delay(1000, -1, 1)); + EXPECT_EQ(0, RateLimiterEntry::compute_delay(1000, -1000, 1)); + + // Ceil division at interval=1: tokens replenish at `limit` per second + EXPECT_EQ(1, RateLimiterEntry::compute_delay(1000, 1, 1)); + EXPECT_EQ(1, RateLimiterEntry::compute_delay(1000, 1000, 1)); + EXPECT_EQ(2, RateLimiterEntry::compute_delay(1000, 1001, 1)); + EXPECT_EQ(2, RateLimiterEntry::compute_delay(1000, 2000, 1)); + EXPECT_EQ(3, RateLimiterEntry::compute_delay(1000, 2001, 1)); + + // Ceil division at interval=100: tokens replenish at `limit` per 100s + EXPECT_EQ(0, RateLimiterEntry::compute_delay(10000, 0, 100)); + EXPECT_EQ(1, RateLimiterEntry::compute_delay(10000, 1, 100)); + EXPECT_EQ(1, RateLimiterEntry::compute_delay(10000, 100, 100)); + EXPECT_EQ(2, RateLimiterEntry::compute_delay(10000, 101, 100)); + EXPECT_EQ(2, RateLimiterEntry::compute_delay(10000, 200, 100)); + EXPECT_EQ(3, RateLimiterEntry::compute_delay(10000, 201, 100)); +} + TEST(RGWRateLimit, op_limit_not_enabled) { // info.enabled = false, so no limit std::atomic_bool replacing; std::condition_variable cv; - RateLimiter ratelimit(replacing, cv); + RateLimiter ratelimit(g_ceph_context, replacing, cv); RGWRateLimitInfo info; auto time = ceph::coarse_real_clock::now(); std::string key = "uuser123"; - bool success = ratelimit.should_rate_limit("PUT", key, time, &info); - EXPECT_EQ(false, success); + int64_t delay = ratelimit.should_rate_limit("PUT", key, time, &info, ""); + EXPECT_EQ(0, delay); } TEST(RGWRateLimit, reject_op_over_limit) { - // check that request is being rejected because there are not enough tokens + // check that request is being rejected because there are not enough tokens, + // and that the returned delay matches the configured interval (default 60s) + // when the user is exactly one token short. std::atomic_bool replacing; std::condition_variable cv; - RateLimiter ratelimit(replacing, cv); + RateLimiter ratelimit(g_ceph_context, replacing, cv); RGWRateLimitInfo info; info.enabled = true; info.max_read_ops = 1; auto time = ceph::coarse_real_clock::now(); std::string key = "uuser123"; - bool success = ratelimit.should_rate_limit("GET", key, time, &info); + int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, ""); time = ceph::coarse_real_clock::now(); - success = ratelimit.should_rate_limit("GET", key, time, &info); - EXPECT_EQ(true, success); + delay = ratelimit.should_rate_limit("GET", key, time, &info, ""); + EXPECT_EQ(60, delay); } TEST(RGWRateLimit, accept_op_after_giveback) { // check that giveback is working fine std::atomic_bool replacing; std::condition_variable cv; - RateLimiter ratelimit(replacing, cv); + RateLimiter ratelimit(g_ceph_context, replacing, cv); RGWRateLimitInfo info; info.enabled = true; info.max_read_ops = 1; auto time = ceph::coarse_real_clock::now(); std::string key = "uuser123"; - bool success = ratelimit.should_rate_limit("GET", key, time, &info); - ratelimit.giveback_tokens("GET", key); + int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, ""); + ratelimit.giveback_tokens("GET", key, "", &info); time = ceph::coarse_real_clock::now(); - success = ratelimit.should_rate_limit("GET", key, time, &info); - EXPECT_EQ(false, success); + delay = ratelimit.should_rate_limit("GET", key, time, &info, ""); + EXPECT_EQ(0, delay); } TEST(RGWRateLimit, accept_op_after_refill) { // check that tokens are being filled properly std::atomic_bool replacing; std::condition_variable cv; - RateLimiter ratelimit(replacing, cv); + RateLimiter ratelimit(g_ceph_context, replacing, cv); RGWRateLimitInfo info; info.enabled = true; info.max_read_ops = 1; auto time = ceph::coarse_real_clock::now(); std::string key = "uuser123"; - bool success = ratelimit.should_rate_limit("GET", key, time, &info); + int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, ""); time += 61s; - success = ratelimit.should_rate_limit("GET", key, time, &info); - EXPECT_EQ(false, success); + delay = ratelimit.should_rate_limit("GET", key, time, &info, ""); + EXPECT_EQ(0, delay); } TEST(RGWRateLimit, reject_bw_over_limit) { // check that a newer request is rejected if there is no enough tokens (bw) std::atomic_bool replacing; std::condition_variable cv; - RateLimiter ratelimit(replacing, cv); + RateLimiter ratelimit(g_ceph_context, replacing, cv); RGWRateLimitInfo info; info.enabled = true; info.max_read_bytes = 1; auto time = ceph::coarse_real_clock::now(); std::string key = "uuser123"; - bool success = ratelimit.should_rate_limit("GET", key, time, &info); + int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, ""); ratelimit.decrease_bytes("GET",key, 2, &info); time = ceph::coarse_real_clock::now(); - success = ratelimit.should_rate_limit("GET", key, time, &info); - EXPECT_EQ(true, success); + delay = ratelimit.should_rate_limit("GET", key, time, &info, ""); + EXPECT_GT(delay, 0); } TEST(RGWRateLimit, accept_bw) { // check that when there are enough tokens (bw) the request is still being served std::atomic_bool replacing; std::condition_variable cv; - RateLimiter ratelimit(replacing, cv); + RateLimiter ratelimit(g_ceph_context, replacing, cv); RGWRateLimitInfo info; info.enabled = true; info.max_read_bytes = 2; auto time = ceph::coarse_real_clock::now(); std::string key = "uuser123"; - bool success = ratelimit.should_rate_limit("GET", key, time, &info); + int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, ""); ratelimit.decrease_bytes("GET",key, 1, &info); time = ceph::coarse_real_clock::now(); - success = ratelimit.should_rate_limit("GET", key, time, &info); - EXPECT_EQ(false, success); + delay = ratelimit.should_rate_limit("GET", key, time, &info, ""); + EXPECT_EQ(0, delay); } TEST(RGWRateLimit, check_bw_debt_at_max_120secs) { // check that the bandwidth debt is not larger than 120 seconds std::atomic_bool replacing; std::condition_variable cv; - RateLimiter ratelimit(replacing, cv); + RateLimiter ratelimit(g_ceph_context, replacing, cv); RGWRateLimitInfo info; info.enabled = true; info.max_read_bytes = 2; auto time = ceph::coarse_real_clock::now(); std::string key = "uuser123"; - bool success = ratelimit.should_rate_limit("GET", key, time, &info); + int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, ""); ratelimit.decrease_bytes("GET",key, 100, &info); time += 121s; - success = ratelimit.should_rate_limit("GET", key, time, &info); - EXPECT_EQ(false, success); + delay = ratelimit.should_rate_limit("GET", key, time, &info, ""); + EXPECT_EQ(0, delay); } TEST(RGWRateLimit, check_that_bw_limit_not_affect_ops) { // check that high read bytes limit, does not affect ops limit std::atomic_bool replacing; std::condition_variable cv; - RateLimiter ratelimit(replacing, cv); + RateLimiter ratelimit(g_ceph_context, replacing, cv); RGWRateLimitInfo info; info.enabled = true; info.max_read_ops = 1; info.max_read_bytes = 100000000; auto time = ceph::coarse_real_clock::now(); std::string key = "uuser123"; - bool success = ratelimit.should_rate_limit("GET", key, time, &info); + int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, ""); ratelimit.decrease_bytes("GET",key, 10000, &info); time = ceph::coarse_real_clock::now(); - success = ratelimit.should_rate_limit("GET", key, time, &info); - EXPECT_EQ(true, success); + delay = ratelimit.should_rate_limit("GET", key, time, &info, ""); + EXPECT_GT(delay, 0); } TEST(RGWRateLimit, read_limit_does_not_affect_writes) { // read limit does not affect writes std::atomic_bool replacing; std::condition_variable cv; - RateLimiter ratelimit(replacing, cv); + RateLimiter ratelimit(g_ceph_context, replacing, cv); RGWRateLimitInfo info; info.enabled = true; info.max_read_ops = 1; info.max_read_bytes = 100000000; auto time = ceph::coarse_real_clock::now(); std::string key = "uuser123"; - bool success = ratelimit.should_rate_limit("PUT", key, time, &info); + int64_t delay = ratelimit.should_rate_limit("PUT", key, time, &info, ""); ratelimit.decrease_bytes("PUT",key, 10000, &info); time = ceph::coarse_real_clock::now(); - success = ratelimit.should_rate_limit("PUT", key, time, &info); - EXPECT_EQ(false, success); + delay = ratelimit.should_rate_limit("PUT", key, time, &info, ""); + EXPECT_EQ(0, delay); } TEST(RGWRateLimit, write_limit_does_not_affect_reads) { // write limit does not affect reads std::atomic_bool replacing; std::condition_variable cv; - RateLimiter ratelimit(replacing, cv); + RateLimiter ratelimit(g_ceph_context, replacing, cv); RGWRateLimitInfo info; info.enabled = true; info.max_write_ops = 1; info.max_write_bytes = 100000000; auto time = ceph::coarse_real_clock::now(); std::string key = "uuser123"; - bool success = ratelimit.should_rate_limit("GET", key, time, &info); + int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, ""); ratelimit.decrease_bytes("GET",key, 10000, &info); time = ceph::coarse_real_clock::now(); - success = ratelimit.should_rate_limit("GET", key, time, &info); - EXPECT_EQ(false, success); + delay = ratelimit.should_rate_limit("GET", key, time, &info, ""); + EXPECT_EQ(0, delay); } TEST(RGWRateLimit, allow_unlimited_access) @@ -179,13 +212,57 @@ TEST(RGWRateLimit, allow_unlimited_access) // 0 values in RGWRateLimitInfo should allow unlimited access std::atomic_bool replacing; std::condition_variable cv; - RateLimiter ratelimit(replacing, cv); + RateLimiter ratelimit(g_ceph_context, replacing, cv); + RGWRateLimitInfo info; + info.enabled = true; + auto time = ceph::coarse_real_clock::now(); + std::string key = "uuser123"; + int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, ""); + EXPECT_EQ(0, delay); +} + +TEST(RGWRateLimit, unlimited_access_not_left_large_read_ops_budget) +{ + // 0 values in RGWRateLimitInfo should allow unlimited access + std::atomic_bool replacing; + std::condition_variable cv; + RateLimiter ratelimit(g_ceph_context, replacing, cv); + RGWRateLimitInfo info; + info.enabled = true; + auto time = ceph::coarse_real_clock::now(); + std::string key = "uuser123"; + + for (int i = 0; i < 10; i++) + { + int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, ""); + EXPECT_EQ(0, delay); + } + time += 61s; + info.max_read_ops = 1; // make read ops limited + int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, ""); + EXPECT_EQ(0, delay); +} + +TEST(RGWRateLimit, unlimited_access_not_left_large_write_ops_budget) +{ + // 0 values in RGWRateLimitInfo should allow unlimited access + std::atomic_bool replacing; + std::condition_variable cv; + RateLimiter ratelimit(g_ceph_context, replacing, cv); RGWRateLimitInfo info; info.enabled = true; auto time = ceph::coarse_real_clock::now(); std::string key = "uuser123"; - bool success = ratelimit.should_rate_limit("GET", key, time, &info); - EXPECT_EQ(false, success); + + for (int i = 0; i < 10; i++) + { + int64_t delay = ratelimit.should_rate_limit("PUT", key, time, &info, ""); + EXPECT_EQ(0, delay); + } + time += 61s; + info.max_write_ops = 1; // make write ops limited + int64_t delay = ratelimit.should_rate_limit("PUT", key, time, &info, ""); + EXPECT_EQ(0, delay); } TEST(RGWRateLimitGC, NO_GC_AHEAD_OF_TIME) @@ -199,7 +276,7 @@ TEST(RGWRateLimitGC, NO_GC_AHEAD_OF_TIME) RGWRateLimitInfo info; auto time = ceph::coarse_real_clock::now(); std::string key = "uuser123"; - active->should_rate_limit("GET", key, time, &info); + active->should_rate_limit("GET", key, time, &info, ""); auto activegc = ratelimit->get_active(); EXPECT_EQ(activegc, active); } @@ -217,7 +294,7 @@ TEST(RGWRateLimiterGC, GC_IS_WORKING) std::string key = "-1"; for(int i = 0; i < 2000000; i++) { - active->should_rate_limit("GET", key, time, &info); + active->should_rate_limit("GET", key, time, &info, ""); key = std::to_string(i); } auto activegc = ratelimit->get_active(); @@ -231,22 +308,23 @@ TEST(RGWRateLimitEntry, op_limit_not_enabled) RateLimiterEntry entry; RGWRateLimitInfo info; auto time = ceph::coarse_real_clock::now().time_since_epoch(); - bool success = entry.should_rate_limit(false, &info, time); - EXPECT_EQ(false, success); + int64_t delay = entry.should_rate_limit(OpType::Read, &info, time); + EXPECT_EQ(0, delay); } TEST(RGWRateLimitEntry, reject_op_over_limit) { - // check that request is being rejected because there are not enough tokens - + // check that request is being rejected because there are not enough tokens, + // and that the returned delay matches the configured interval (default 60s) + // when the user is exactly one token short. RGWRateLimitInfo info; RateLimiterEntry entry; info.enabled = true; info.max_read_ops = 1; auto time = ceph::coarse_real_clock::now().time_since_epoch(); - bool success = entry.should_rate_limit(true, &info, time); + int64_t delay = entry.should_rate_limit(OpType::Read, &info, time); time = ceph::coarse_real_clock::now().time_since_epoch(); - success = entry.should_rate_limit(true, &info, time); - EXPECT_EQ(true, success); + delay = entry.should_rate_limit(OpType::Read, &info, time); + EXPECT_EQ(60, delay); } TEST(RGWRateLimitEntry, accept_op_after_giveback) { @@ -256,11 +334,11 @@ TEST(RGWRateLimitEntry, accept_op_after_giveback) info.enabled = true; info.max_read_ops = 1; auto time = ceph::coarse_real_clock::now().time_since_epoch(); - bool success = entry.should_rate_limit(true, &info, time); - entry.giveback_tokens(true); + int64_t delay = entry.should_rate_limit(OpType::Read, &info, time); + entry.giveback_tokens(OpType::Read); time = ceph::coarse_real_clock::now().time_since_epoch(); - success = entry.should_rate_limit(true, &info, time); - EXPECT_EQ(false, success); + delay = entry.should_rate_limit(OpType::Read, &info, time); + EXPECT_EQ(0, delay); } TEST(RGWRateLimitEntry, accept_op_after_refill) { @@ -270,10 +348,10 @@ TEST(RGWRateLimitEntry, accept_op_after_refill) info.enabled = true; info.max_read_ops = 1; auto time = ceph::coarse_real_clock::now().time_since_epoch(); - bool success = entry.should_rate_limit(true, &info, time); + int64_t delay = entry.should_rate_limit(OpType::Read, &info, time); time += 61s; - success = entry.should_rate_limit(true, &info, time); - EXPECT_EQ(false, success); + delay = entry.should_rate_limit(OpType::Read, &info, time); + EXPECT_EQ(0, delay); } TEST(RGWRateLimitEntry, reject_bw_over_limit) { @@ -283,11 +361,11 @@ TEST(RGWRateLimitEntry, reject_bw_over_limit) info.enabled = true; info.max_read_bytes = 1; auto time = ceph::coarse_real_clock::now().time_since_epoch(); - bool success = entry.should_rate_limit(true, &info, time); + int64_t delay = entry.should_rate_limit(OpType::Read, &info, time); entry.decrease_bytes(true, 2, &info); time = ceph::coarse_real_clock::now().time_since_epoch(); - success = entry.should_rate_limit(true, &info, time); - EXPECT_EQ(true, success); + delay = entry.should_rate_limit(OpType::Read, &info, time); + EXPECT_GT(delay, 0); } TEST(RGWRateLimitEntry, accept_bw) { @@ -297,11 +375,11 @@ TEST(RGWRateLimitEntry, accept_bw) info.enabled = true; info.max_read_bytes = 2; auto time = ceph::coarse_real_clock::now().time_since_epoch(); - bool success = entry.should_rate_limit(true, &info, time); + int64_t delay = entry.should_rate_limit(OpType::Read, &info, time); entry.decrease_bytes(true, 1, &info); time = ceph::coarse_real_clock::now().time_since_epoch(); - success = entry.should_rate_limit(true, &info, time); - EXPECT_EQ(false, success); + delay = entry.should_rate_limit(OpType::Read, &info, time); + EXPECT_EQ(0, delay); } TEST(RGWRateLimitEntry, check_bw_debt_at_max_120secs) { @@ -311,11 +389,11 @@ TEST(RGWRateLimitEntry, check_bw_debt_at_max_120secs) info.enabled = true; info.max_read_bytes = 2; auto time = ceph::coarse_real_clock::now().time_since_epoch(); - bool success = entry.should_rate_limit(true, &info, time); + int64_t delay = entry.should_rate_limit(OpType::Read, &info, time); entry.decrease_bytes(true, 100, &info); time += 121s; - success = entry.should_rate_limit(true, &info, time); - EXPECT_EQ(false, success); + delay = entry.should_rate_limit(OpType::Read, &info, time); + EXPECT_EQ(0, delay); } TEST(RGWRateLimitEntry, check_that_bw_limit_not_affect_ops) { @@ -326,11 +404,11 @@ TEST(RGWRateLimitEntry, check_that_bw_limit_not_affect_ops) info.max_read_ops = 1; info.max_read_bytes = 100000000; auto time = ceph::coarse_real_clock::now().time_since_epoch(); - bool success = entry.should_rate_limit(true, &info, time); + int64_t delay = entry.should_rate_limit(OpType::Read, &info, time); entry.decrease_bytes(true, 10000, &info); time = ceph::coarse_real_clock::now().time_since_epoch(); - success = entry.should_rate_limit(true, &info, time); - EXPECT_EQ(true, success); + delay = entry.should_rate_limit(OpType::Read, &info, time); + EXPECT_GT(delay, 0); } TEST(RGWRateLimitEntry, read_limit_does_not_affect_writes) { @@ -341,11 +419,11 @@ TEST(RGWRateLimitEntry, read_limit_does_not_affect_writes) info.max_read_ops = 1; info.max_read_bytes = 100000000; auto time = ceph::coarse_real_clock::now().time_since_epoch(); - bool success = entry.should_rate_limit(false, &info, time); + int64_t delay = entry.should_rate_limit(OpType::Write, &info, time); entry.decrease_bytes(false, 10000, &info); time = ceph::coarse_real_clock::now().time_since_epoch(); - success = entry.should_rate_limit(false, &info, time); - EXPECT_EQ(false, success); + delay = entry.should_rate_limit(OpType::Write, &info, time); + EXPECT_EQ(0, delay); } TEST(RGWRateLimitEntry, write_limit_does_not_affect_reads) { @@ -357,11 +435,11 @@ TEST(RGWRateLimitEntry, write_limit_does_not_affect_reads) info.max_write_bytes = 100000000; auto time = ceph::coarse_real_clock::now().time_since_epoch(); std::string key = "uuser123"; - bool success = entry.should_rate_limit(true, &info, time); + int64_t delay = entry.should_rate_limit(OpType::Read, &info, time); entry.decrease_bytes(true, 10000, &info); time = ceph::coarse_real_clock::now().time_since_epoch(); - success = entry.should_rate_limit(true, &info, time); - EXPECT_EQ(false, success); + delay = entry.should_rate_limit(OpType::Read, &info, time); + EXPECT_EQ(0, delay); } TEST(RGWRateLimitEntry, allow_unlimited_access) @@ -371,6 +449,686 @@ TEST(RGWRateLimitEntry, allow_unlimited_access) RGWRateLimitInfo info; info.enabled = true; auto time = ceph::coarse_real_clock::now().time_since_epoch(); - bool success = entry.should_rate_limit(true, &info, time); - EXPECT_EQ(false, success); + int64_t delay = entry.should_rate_limit(OpType::Read, &info, time); + EXPECT_EQ(0, delay); +} + + +// Helpers for LIST op resource string +// ref uri: "/aaa-gonzo-staging-bbb-checkpoint1-us-west-0000?list-type=2&delimiter=%2F&max-keys=2&prefix=spark%2Fgonzo-avro%2Fsplunk_hec_test%2Fchunk-commits%2F%2F00000007999&encoding-type=url"; +const std::string RES_LIST_TYPE_2 = "?list-type=2"; +const std::string RES_DELIMITER = "&delimiter=%2F"; +const std::string RES_PREFIX = "&prefix=spark%2Fgonzo-avro%2Fsplunk_hec_test%2Fchunk-commits%2F%2F00000007999"; + +TEST(RGWRateLimit, reject_list_op_over_limit) +{ + // check that LIST op is being rejected because there are not enough tokens + std::atomic_bool replacing; + std::condition_variable cv; + RateLimiter ratelimit(g_ceph_context, replacing, cv); + RGWRateLimitInfo info; + info.enabled = true; + info.max_list_ops = 1; + auto time = ceph::coarse_real_clock::now(); + std::string key = "uuser_list"; + int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2); + time = ceph::coarse_real_clock::now(); + delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2); + EXPECT_GT(delay, 0); +} + +TEST(RGWRateLimit, accept_list_op_after_giveback) +{ + // check that giveback is working for LIST ops + std::atomic_bool replacing; + std::condition_variable cv; + RateLimiter ratelimit(g_ceph_context, replacing, cv); + RGWRateLimitInfo info; + info.enabled = true; + info.max_list_ops = 1; + auto time = ceph::coarse_real_clock::now(); + std::string key = "uuser_list"; + int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2); + ratelimit.giveback_tokens("GET", key, RES_LIST_TYPE_2, &info); + time = ceph::coarse_real_clock::now(); + delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2); + EXPECT_EQ(0, delay); +} + +TEST(RGWRateLimit, accept_list_op_after_refill) +{ + // check that tokens are being filled properly for LIST ops + std::atomic_bool replacing; + std::condition_variable cv; + RateLimiter ratelimit(g_ceph_context, replacing, cv); + RGWRateLimitInfo info; + info.enabled = true; + info.max_list_ops = 1; + auto time = ceph::coarse_real_clock::now(); + std::string key = "uuser_list"; + int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2); + time += 61s; + delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2); + EXPECT_EQ(0, delay); +} + +TEST(RGWRateLimit, list_limit_does_not_affect_reads) +{ + // list limit does not affect reads + std::atomic_bool replacing; + std::condition_variable cv; + RateLimiter ratelimit(g_ceph_context, replacing, cv); + RGWRateLimitInfo info; + info.enabled = true; + info.max_list_ops = 1; + info.max_read_ops = 1; + auto time = ceph::coarse_real_clock::now(); + std::string key = "uuser_list"; + int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2); + // Should still be able to do a normal GET (read) + delay = ratelimit.should_rate_limit("GET", key, time, &info, ""); + EXPECT_EQ(0, delay); +} + +TEST(RGWRateLimit, read_limit_does_not_affect_lists) +{ + // read limit does not affect lists + std::atomic_bool replacing; + std::condition_variable cv; + RateLimiter ratelimit(g_ceph_context, replacing, cv); + RGWRateLimitInfo info; + info.enabled = true; + info.max_list_ops = 1; + info.max_read_ops = 1; + auto time = ceph::coarse_real_clock::now(); + std::string key = "uuser_list"; + int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, ""); + // Should still be able to do a LIST op + delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2); + EXPECT_EQ(0, delay); +} + +TEST(RGWRateLimit, list_limit_does_not_affect_writes) +{ + // list limit does not affect writes + std::atomic_bool replacing; + std::condition_variable cv; + RateLimiter ratelimit(g_ceph_context, replacing, cv); + RGWRateLimitInfo info; + info.enabled = true; + info.max_list_ops = 1; + info.max_write_ops = 1; + auto time = ceph::coarse_real_clock::now(); + std::string key = "uuser_list"; + int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2); + // Should still be able to do a PUT (write) + delay = ratelimit.should_rate_limit("PUT", key, time, &info, ""); + EXPECT_EQ(0, delay); +} + +TEST(RGWRateLimit, unlimited_access_not_left_large_list_ops_budget) +{ + // 0 values in RGWRateLimitInfo should allow unlimited access + std::atomic_bool replacing; + std::condition_variable cv; + RateLimiter ratelimit(g_ceph_context, replacing, cv); + RGWRateLimitInfo info; + info.enabled = true; + auto time = ceph::coarse_real_clock::now(); + std::string key = "uuser_list"; + + for (int i = 0; i < 10; i++) + { + int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2); + EXPECT_EQ(0, delay); + } + time += 61s; + info.max_list_ops = 1; // make list ops limited + int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2); + EXPECT_EQ(0, delay); +} + +TEST(RGWRateLimit, write_limit_does_not_affect_lists) +{ + // write limit does not affect lists + std::atomic_bool replacing; + std::condition_variable cv; + RateLimiter ratelimit(g_ceph_context, replacing, cv); + RGWRateLimitInfo info; + info.enabled = true; + info.max_list_ops = 1; + info.max_write_ops = 1; + auto time = ceph::coarse_real_clock::now(); + std::string key = "uuser_list"; + int64_t delay = ratelimit.should_rate_limit("PUT", key, time, &info, ""); + // Should still be able to do a LIST op + delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2); + EXPECT_EQ(0, delay); +} + +TEST(RGWRateLimit, list_limit_does_not_affect_deletes) +{ + // list limit does not affect deletes + std::atomic_bool replacing; + std::condition_variable cv; + RateLimiter ratelimit(g_ceph_context, replacing, cv); + RGWRateLimitInfo info; + info.enabled = true; + info.max_list_ops = 1; + info.max_delete_ops = 1; + auto time = ceph::coarse_real_clock::now(); + std::string key = "uuser_list"; + int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2); + // Should still be able to do a DELETE op + delay = ratelimit.should_rate_limit("DELETE", key, time, &info, ""); + EXPECT_EQ(0, delay); +} + +TEST(RGWRateLimit, delete_limit_does_not_affect_lists) +{ + // delete limit does not affect lists + std::atomic_bool replacing; + std::condition_variable cv; + RateLimiter ratelimit(g_ceph_context, replacing, cv); + RGWRateLimitInfo info; + info.enabled = true; + info.max_list_ops = 1; + info.max_delete_ops = 1; + auto time = ceph::coarse_real_clock::now(); + std::string key = "uuser_list"; + int64_t delay = ratelimit.should_rate_limit("DELETE", key, time, &info, ""); + // Should still be able to do a LIST op + delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2); + EXPECT_EQ(0, delay); +} + +// LIST RES_DELIMITER minimal tests +TEST(RGWRateLimit, reject_delimiter_op_over_limit) +{ + // check that DELIMITER op is being rejected because there are not enough tokens + std::atomic_bool replacing; + std::condition_variable cv; + RateLimiter ratelimit(g_ceph_context, replacing, cv); + RGWRateLimitInfo info; + info.enabled = true; + info.max_list_ops = 1; + auto time = ceph::coarse_real_clock::now(); + std::string key = "uuser_list"; + int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_DELIMITER); + time = ceph::coarse_real_clock::now(); + delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_DELIMITER); + EXPECT_GT(delay, 0); +} + +TEST(RGWRateLimit, accept_delimiter_op_after_giveback) +{ + // check that giveback is working for DELIMITER ops + std::atomic_bool replacing; + std::condition_variable cv; + RateLimiter ratelimit(g_ceph_context, replacing, cv); + RGWRateLimitInfo info; + info.enabled = true; + info.max_list_ops = 1; + auto time = ceph::coarse_real_clock::now(); + std::string key = "uuser_list"; + int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_DELIMITER); + ratelimit.giveback_tokens("GET", key, RES_DELIMITER, &info); + time = ceph::coarse_real_clock::now(); + delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_DELIMITER); + EXPECT_EQ(0, delay); +} + +TEST(RGWRateLimit, accept_delimiter_op_after_refill) +{ + // check that tokens are being filled properly for DELIMITER ops + std::atomic_bool replacing; + std::condition_variable cv; + RateLimiter ratelimit(g_ceph_context, replacing, cv); + RGWRateLimitInfo info; + info.enabled = true; + info.max_list_ops = 1; + auto time = ceph::coarse_real_clock::now(); + std::string key = "uuser_list"; + int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_DELIMITER); + time += 61s; + delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_DELIMITER); + EXPECT_EQ(0, delay); +} + +// LIST RES_PREFIX minimal tests +TEST(RGWRateLimit, reject_prefix_op_over_limit) +{ + // check that PREFIX op is being rejected because there are not enough tokens + std::atomic_bool replacing; + std::condition_variable cv; + RateLimiter ratelimit(g_ceph_context, replacing, cv); + RGWRateLimitInfo info; + info.enabled = true; + info.max_list_ops = 1; + auto time = ceph::coarse_real_clock::now(); + std::string key = "uuser_list"; + int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_PREFIX); + time = ceph::coarse_real_clock::now(); + delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_PREFIX); + EXPECT_GT(delay, 0); +} + +TEST(RGWRateLimit, accept_prefix_op_after_giveback) +{ + // check that giveback is working for PREFIX ops + std::atomic_bool replacing; + std::condition_variable cv; + RateLimiter ratelimit(g_ceph_context, replacing, cv); + RGWRateLimitInfo info; + info.enabled = true; + info.max_list_ops = 1; + auto time = ceph::coarse_real_clock::now(); + std::string key = "uuser_list"; + int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_PREFIX); + ratelimit.giveback_tokens("GET", key, RES_PREFIX, &info); + time = ceph::coarse_real_clock::now(); + delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_PREFIX); + EXPECT_EQ(0, delay); +} + +TEST(RGWRateLimit, accept_prefix_op_after_refill) +{ + // check that tokens are being filled properly for PREFIX ops + std::atomic_bool replacing; + std::condition_variable cv; + RateLimiter ratelimit(g_ceph_context, replacing, cv); + RGWRateLimitInfo info; + info.enabled = true; + info.max_list_ops = 1; + auto time = ceph::coarse_real_clock::now(); + std::string key = "uuser_list"; + int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_PREFIX); + time += 61s; + delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_PREFIX); + EXPECT_EQ(0, delay); +} + + +TEST(RGWRateLimitEntry, reject_list_op_over_limit) +{ + // check that LIST request is being rejected because there are not enough tokens + RGWRateLimitInfo info; + RateLimiterEntry entry; + info.enabled = true; + info.max_list_ops = 1; + auto time = ceph::coarse_real_clock::now().time_since_epoch(); + int64_t delay = entry.should_rate_limit(OpType::List, &info, time); + time = ceph::coarse_real_clock::now().time_since_epoch(); + delay = entry.should_rate_limit(OpType::List, &info, time); + EXPECT_GT(delay, 0); } + +TEST(RGWRateLimitEntry, accept_list_op_after_giveback) +{ + // check that giveback is working fine for LIST ops + RGWRateLimitInfo info; + RateLimiterEntry entry; + info.enabled = true; + info.max_list_ops = 1; + auto time = ceph::coarse_real_clock::now().time_since_epoch(); + int64_t delay = entry.should_rate_limit(OpType::List, &info, time); + entry.giveback_tokens(OpType::List); + time = ceph::coarse_real_clock::now().time_since_epoch(); + delay = entry.should_rate_limit(OpType::List, &info, time); + EXPECT_EQ(0, delay); +} + +TEST(RGWRateLimitEntry, accept_list_op_after_refill) +{ + // check that tokens are being filled properly for LIST ops + RateLimiterEntry entry; + RGWRateLimitInfo info; + info.enabled = true; + info.max_list_ops = 1; + auto time = ceph::coarse_real_clock::now().time_since_epoch(); + int64_t delay = entry.should_rate_limit(OpType::List, &info, time); + time += 61s; + delay = entry.should_rate_limit(OpType::List, &info, time); + EXPECT_EQ(0, delay); +} + +TEST(RGWRateLimitEntry, list_limit_does_not_affect_reads) +{ + // list limit does not affect reads + RateLimiterEntry entry; + RGWRateLimitInfo info; + info.enabled = true; + info.max_list_ops = 1; + info.max_read_ops = 1; + auto time = ceph::coarse_real_clock::now().time_since_epoch(); + int64_t delay = entry.should_rate_limit(OpType::List, &info, time); + time = ceph::coarse_real_clock::now().time_since_epoch(); + delay = entry.should_rate_limit(OpType::Read, &info, time); + EXPECT_EQ(0, delay); +} + +TEST(RGWRateLimitEntry, read_limit_does_not_affect_lists) +{ + // read limit does not affect lists + RateLimiterEntry entry; + RGWRateLimitInfo info; + info.enabled = true; + info.max_list_ops = 1; + info.max_read_ops = 1; + auto time = ceph::coarse_real_clock::now().time_since_epoch(); + int64_t delay = entry.should_rate_limit(OpType::Read, &info, time); + time = ceph::coarse_real_clock::now().time_since_epoch(); + delay = entry.should_rate_limit(OpType::List, &info, time); + EXPECT_EQ(0, delay); +} + +TEST(RGWRateLimitEntry, list_limit_does_not_affect_writes) +{ + // list limit does not affect writes + RateLimiterEntry entry; + RGWRateLimitInfo info; + info.enabled = true; + info.max_list_ops = 1; + info.max_write_ops = 1; + auto time = ceph::coarse_real_clock::now().time_since_epoch(); + int64_t delay = entry.should_rate_limit(OpType::List, &info, time); + time = ceph::coarse_real_clock::now().time_since_epoch(); + delay = entry.should_rate_limit(OpType::Write, &info, time); + EXPECT_EQ(0, delay); +} + +TEST(RGWRateLimitEntry, write_limit_does_not_affect_lists) +{ + // write limit does not affect lists + RateLimiterEntry entry; + RGWRateLimitInfo info; + info.enabled = true; + info.max_list_ops = 1; + info.max_write_ops = 1; + auto time = ceph::coarse_real_clock::now().time_since_epoch(); + int64_t delay = entry.should_rate_limit(OpType::Write, &info, time); + time = ceph::coarse_real_clock::now().time_since_epoch(); + delay = entry.should_rate_limit(OpType::List, &info, time); + EXPECT_EQ(0, delay); +} + +TEST(RGWRateLimitEntry, list_limit_does_not_affect_deletes) +{ + // list limit does not affect deletes + RateLimiterEntry entry; + RGWRateLimitInfo info; + info.enabled = true; + info.max_list_ops = 1; + info.max_delete_ops = 1; + auto time = ceph::coarse_real_clock::now().time_since_epoch(); + int64_t delay = entry.should_rate_limit(OpType::List, &info, time); + time = ceph::coarse_real_clock::now().time_since_epoch(); + delay = entry.should_rate_limit(OpType::Delete, &info, time); + EXPECT_EQ(0, delay); +} + +TEST(RGWRateLimitEntry, delete_limit_does_not_affect_lists) +{ + // delete limit does not affect lists + RateLimiterEntry entry; + RGWRateLimitInfo info; + info.enabled = true; + info.max_list_ops = 1; + info.max_delete_ops = 1; + auto time = ceph::coarse_real_clock::now().time_since_epoch(); + int64_t delay = entry.should_rate_limit(OpType::Delete, &info, time); + time = ceph::coarse_real_clock::now().time_since_epoch(); + delay = entry.should_rate_limit(OpType::List, &info, time); + EXPECT_EQ(0, delay); +} + + +TEST(RGWRateLimit, reject_delete_op_over_limit) +{ + // check that DELETE op is being rejected because there are not enough tokens + std::atomic_bool replacing; + std::condition_variable cv; + RateLimiter ratelimit(g_ceph_context, replacing, cv); + RGWRateLimitInfo info; + info.enabled = true; + info.max_delete_ops = 1; + auto time = ceph::coarse_real_clock::now(); + std::string key = "uuser_delete"; + int64_t delay = ratelimit.should_rate_limit("DELETE", key, time, &info, ""); + time = ceph::coarse_real_clock::now(); + delay = ratelimit.should_rate_limit("DELETE", key, time, &info, ""); + EXPECT_GT(delay, 0); +} + +TEST(RGWRateLimit, accept_delete_op_after_giveback) +{ + // check that giveback is working for DELETE ops + std::atomic_bool replacing; + std::condition_variable cv; + RateLimiter ratelimit(g_ceph_context, replacing, cv); + RGWRateLimitInfo info; + info.enabled = true; + info.max_delete_ops = 1; + auto time = ceph::coarse_real_clock::now(); + std::string key = "uuser_delete"; + int64_t delay = ratelimit.should_rate_limit("DELETE", key, time, &info, ""); + ratelimit.giveback_tokens("DELETE", key, "", &info); + time = ceph::coarse_real_clock::now(); + delay = ratelimit.should_rate_limit("DELETE", key, time, &info, ""); + EXPECT_EQ(0, delay); +} + +TEST(RGWRateLimit, accept_delete_op_after_refill) +{ + // check that tokens are being filled properly for DELETE ops + std::atomic_bool replacing; + std::condition_variable cv; + RateLimiter ratelimit(g_ceph_context, replacing, cv); + RGWRateLimitInfo info; + info.enabled = true; + info.max_delete_ops = 1; + auto time = ceph::coarse_real_clock::now(); + std::string key = "uuser_delete"; + int64_t delay = ratelimit.should_rate_limit("DELETE", key, time, &info, ""); + time += 61s; + delay = ratelimit.should_rate_limit("DELETE", key, time, &info, ""); + EXPECT_EQ(0, delay); +} + +TEST(RGWRateLimit, delete_limit_does_not_affect_reads) +{ + // delete limit does not affect reads + std::atomic_bool replacing; + std::condition_variable cv; + RateLimiter ratelimit(g_ceph_context, replacing, cv); + RGWRateLimitInfo info; + info.enabled = true; + info.max_delete_ops = 1; + info.max_read_ops = 1; + auto time = ceph::coarse_real_clock::now(); + std::string key = "uuser_delete"; + int64_t delay = ratelimit.should_rate_limit("DELETE", key, time, &info, ""); + // Should still be able to do a normal GET (read) + delay = ratelimit.should_rate_limit("GET", key, time, &info, ""); + EXPECT_EQ(0, delay); +} + +TEST(RGWRateLimit, read_limit_does_not_affect_deletes) +{ + // read limit does not affect deletes + std::atomic_bool replacing; + std::condition_variable cv; + RateLimiter ratelimit(g_ceph_context, replacing, cv); + RGWRateLimitInfo info; + info.enabled = true; + info.max_delete_ops = 1; + info.max_read_ops = 1; + auto time = ceph::coarse_real_clock::now(); + std::string key = "uuser_delete"; + int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, ""); + // Should still be able to do a DELETE op + delay = ratelimit.should_rate_limit("DELETE", key, time, &info, ""); + EXPECT_EQ(0, delay); +} + +TEST(RGWRateLimit, write_limit_does_not_affect_deletes) +{ + // write limit does not affect deletes + std::atomic_bool replacing; + std::condition_variable cv; + RateLimiter ratelimit(g_ceph_context, replacing, cv); + RGWRateLimitInfo info; + info.enabled = true; + info.max_delete_ops = 1; + info.max_write_ops = 1; + auto time = ceph::coarse_real_clock::now(); + std::string key = "uuser_delete"; + int64_t delay = ratelimit.should_rate_limit("PUT", key, time, &info, ""); + // Should still be able to do a DELETE op + delay = ratelimit.should_rate_limit("DELETE", key, time, &info, ""); + EXPECT_EQ(0, delay); +} + +TEST(RGWRateLimit, delete_limit_does_not_affect_writes) +{ + // delete limit does not affect writes + std::atomic_bool replacing; + std::condition_variable cv; + RateLimiter ratelimit(g_ceph_context, replacing, cv); + RGWRateLimitInfo info; + info.enabled = true; + info.max_delete_ops = 1; + info.max_write_ops = 1; + auto time = ceph::coarse_real_clock::now(); + std::string key = "uuser_delete"; + int64_t delay = ratelimit.should_rate_limit("DELETE", key, time, &info, ""); + // Should still be able to do a PUT (write) + delay = ratelimit.should_rate_limit("PUT", key, time, &info, ""); + EXPECT_EQ(0, delay); +} + +TEST(RGWRateLimit, unlimited_access_not_left_large_delete_ops_budget) +{ + // 0 values in RGWRateLimitInfo should allow unlimited access + std::atomic_bool replacing; + std::condition_variable cv; + RateLimiter ratelimit(g_ceph_context, replacing, cv); + RGWRateLimitInfo info; + info.enabled = true; + auto time = ceph::coarse_real_clock::now(); + std::string key = "uuser_delete"; + + for (int i = 0; i < 10; i++) + { + int64_t delay = ratelimit.should_rate_limit("DELETE", key, time, &info, ""); + EXPECT_EQ(0, delay); + } + time += 61s; + info.max_delete_ops = 1; // make delete ops limited + int64_t delay = ratelimit.should_rate_limit("DELETE", key, time, &info, ""); + EXPECT_EQ(0, delay); +} + +TEST(RGWRateLimitEntry, reject_delete_op_over_limit) +{ + // check that DELETE request is being rejected because there are not enough tokens + RGWRateLimitInfo info; + RateLimiterEntry entry; + info.enabled = true; + info.max_delete_ops = 1; + auto time = ceph::coarse_real_clock::now().time_since_epoch(); + int64_t delay = entry.should_rate_limit(OpType::Delete, &info, time); + time = ceph::coarse_real_clock::now().time_since_epoch(); + delay = entry.should_rate_limit(OpType::Delete, &info, time); + EXPECT_GT(delay, 0); +} + +TEST(RGWRateLimitEntry, accept_delete_op_after_giveback) +{ + // check that giveback is working fine for DELETE ops + RGWRateLimitInfo info; + RateLimiterEntry entry; + info.enabled = true; + info.max_delete_ops = 1; + auto time = ceph::coarse_real_clock::now().time_since_epoch(); + int64_t delay = entry.should_rate_limit(OpType::Delete, &info, time); + entry.giveback_tokens(OpType::Delete); + time = ceph::coarse_real_clock::now().time_since_epoch(); + delay = entry.should_rate_limit(OpType::Delete, &info, time); + EXPECT_EQ(0, delay); +} + +TEST(RGWRateLimitEntry, accept_delete_op_after_refill) +{ + // check that tokens are being filled properly for DELETE ops + RateLimiterEntry entry; + RGWRateLimitInfo info; + info.enabled = true; + info.max_delete_ops = 1; + auto time = ceph::coarse_real_clock::now().time_since_epoch(); + int64_t delay = entry.should_rate_limit(OpType::Delete, &info, time); + time += 61s; + delay = entry.should_rate_limit(OpType::Delete, &info, time); + EXPECT_EQ(0, delay); +} + +TEST(RGWRateLimitEntry, delete_limit_does_not_affect_reads) +{ + // delete limit does not affect reads + RateLimiterEntry entry; + RGWRateLimitInfo info; + info.enabled = true; + info.max_delete_ops = 1; + info.max_read_ops = 1; + auto time = ceph::coarse_real_clock::now().time_since_epoch(); + int64_t delay = entry.should_rate_limit(OpType::Delete, &info, time); + time = ceph::coarse_real_clock::now().time_since_epoch(); + delay = entry.should_rate_limit(OpType::Read, &info, time); + EXPECT_EQ(0, delay); +} + +TEST(RGWRateLimitEntry, read_limit_does_not_affect_deletes) +{ + // read limit does not affect deletes + RateLimiterEntry entry; + RGWRateLimitInfo info; + info.enabled = true; + info.max_delete_ops = 1; + info.max_read_ops = 1; + auto time = ceph::coarse_real_clock::now().time_since_epoch(); + int64_t delay = entry.should_rate_limit(OpType::Read, &info, time); + time = ceph::coarse_real_clock::now().time_since_epoch(); + delay = entry.should_rate_limit(OpType::Delete, &info, time); + EXPECT_EQ(0, delay); +} + +TEST(RGWRateLimitEntry, write_limit_does_not_affect_deletes) +{ + // write limit does not affect deletes + RateLimiterEntry entry; + RGWRateLimitInfo info; + info.enabled = true; + info.max_delete_ops = 1; + info.max_write_ops = 1; + auto time = ceph::coarse_real_clock::now().time_since_epoch(); + int64_t delay = entry.should_rate_limit(OpType::Write, &info, time); + time = ceph::coarse_real_clock::now().time_since_epoch(); + delay = entry.should_rate_limit(OpType::Delete, &info, time); + EXPECT_EQ(0, delay); +} + +TEST(RGWRateLimitEntry, delete_limit_does_not_affect_writes) +{ + // delete limit does not affect writes + RateLimiterEntry entry; + RGWRateLimitInfo info; + info.enabled = true; + info.max_delete_ops = 1; + info.max_write_ops = 1; + auto time = ceph::coarse_real_clock::now().time_since_epoch(); + int64_t delay = entry.should_rate_limit(OpType::Delete, &info, time); + time = ceph::coarse_real_clock::now().time_since_epoch(); + delay = entry.should_rate_limit(OpType::Write, &info, time); + EXPECT_EQ(0, delay); +} +