From 215a061d7fe6bd2d45631cffdee6ad623570ff9a Mon Sep 17 00:00:00 2001 From: jkoberg Date: Mon, 25 Nov 2024 17:19:11 +0100 Subject: [PATCH 1/7] feat(testsuite): use async propagation Signed-off-by: jkoberg --- .drone.star | 1 + 1 file changed, 1 insertion(+) diff --git a/.drone.star b/.drone.star index 84fca6d7e93..32d1abc0ac1 100644 --- a/.drone.star +++ b/.drone.star @@ -2322,6 +2322,7 @@ def ocisServer(storage = "ocis", accounts_hash_difficulty = 4, volumes = [], dep "OCIS_JWT_SECRET": "some-ocis-jwt-secret", "EVENTHISTORY_STORE": "memory", "OCIS_TRANSLATION_PATH": "%s/tests/config/translations" % dirs["base"], + "OCIS_DECOMPOSEDFS_PROPAGATOR": "async", } if deploy_type == "": From 3af3288767b4bd8b112edceaa4780706a00e2a9e Mon Sep 17 00:00:00 2001 From: jkoberg Date: Tue, 26 Nov 2024 11:18:49 +0100 Subject: [PATCH 2/7] feat(testsuite): use propagation delay of 100ms Signed-off-by: jkoberg --- .drone.star | 1 + 1 file changed, 1 insertion(+) diff --git a/.drone.star b/.drone.star index 32d1abc0ac1..a0781f2b57a 100644 --- a/.drone.star +++ b/.drone.star @@ -2323,6 +2323,7 @@ def ocisServer(storage = "ocis", accounts_hash_difficulty = 4, volumes = [], dep "EVENTHISTORY_STORE": "memory", "OCIS_TRANSLATION_PATH": "%s/tests/config/translations" % dirs["base"], "OCIS_DECOMPOSEDFS_PROPAGATOR": "async", + "STORAGE_USERS_ASYNC_PROPAGATOR_PROPAGATION_DELAY": "100ms", } if deploy_type == "": From fe1a0eef8d4e33cc15a48ab7e952d20f820990d0 Mon Sep 17 00:00:00 2001 From: Saw-jan Date: Tue, 26 Nov 2024 16:35:26 +0545 Subject: [PATCH 3/7] test: wait for async propagation if enabled --- .drone.star | 4 ++ .../TestHelpers/HttpRequestHelper.php | 5 ++ tests/acceptance/TestHelpers/WebDavHelper.php | 51 +++++++++++++++++++ 3 files changed, 60 insertions(+) diff --git a/.drone.star b/.drone.star index a0781f2b57a..238bac4fa7b 100644 --- a/.drone.star +++ b/.drone.star @@ -1029,6 +1029,8 @@ def localApiTests(ctx, name, suites, storage = "ocis", extra_environment = {}, w "UPLOAD_DELETE_WAIT_TIME": "1" if storage == "owncloud" else 0, "OCIS_WRAPPER_URL": "http://%s:5200" % OCIS_SERVER_NAME, "WITH_REMOTE_PHP": with_remote_php, + "ASYNC_PROPAGATION": "true", + "ASYNC_PROPAGATION_DELAY_MS": "100", } for item in extra_environment: @@ -1228,6 +1230,8 @@ def coreApiTests(ctx, part_number = 1, number_of_parts = 1, with_remote_php = Fa "UPLOAD_DELETE_WAIT_TIME": "1" if storage == "owncloud" else 0, "OCIS_WRAPPER_URL": "http://%s:5200" % OCIS_SERVER_NAME, "WITH_REMOTE_PHP": with_remote_php, + "ASYNC_PROPAGATION": "true", + "ASYNC_PROPAGATION_DELAY_MS": "100", }, "commands": [ # merge the expected failures diff --git a/tests/acceptance/TestHelpers/HttpRequestHelper.php b/tests/acceptance/TestHelpers/HttpRequestHelper.php index 21fa4f3024f..2e38a122569 100644 --- a/tests/acceptance/TestHelpers/HttpRequestHelper.php +++ b/tests/acceptance/TestHelpers/HttpRequestHelper.php @@ -159,6 +159,11 @@ public static function sendRequestOnce( } HttpLogger::logResponse($response); + + if (WebdavHelper::asyncPropagation()) { + WebdavHelper::waitAsyncPropagationAfterRequest($url, $method, $response->getStatusCode()); + } + return $response; } diff --git a/tests/acceptance/TestHelpers/WebDavHelper.php b/tests/acceptance/TestHelpers/WebDavHelper.php index 77bd99b47f3..ea85e0195a1 100644 --- a/tests/acceptance/TestHelpers/WebDavHelper.php +++ b/tests/acceptance/TestHelpers/WebDavHelper.php @@ -28,6 +28,7 @@ use PHPUnit\Framework\Assert; use Psr\Http\Message\ResponseInterface; use Psr\Http\Message\StreamInterface; +use Symfony\Component\HttpFoundation\Response as HttpResponse; use DateTime; /** @@ -54,6 +55,27 @@ public static function withRemotePhp(): bool { return \getenv("WITH_REMOTE_PHP") !== "false"; } + /** + * @return bool + */ + public static function asyncPropagation(): bool { + return \getenv("ASYNC_PROPAGATION") === "true"; + } + + /** + * async propagation delay in milliseconds + * + * @return int + */ + public static function asyncPropagationDelay(): int { + $delay = (int) \getenv("ASYNC_PROPAGATION_DELAY_MS"); + if ($delay === 0) { + // default delay: 100ms + $delay = 100; + } + return $delay; + } + /** * @param string $urlPath * @@ -76,6 +98,35 @@ public static function isDAVRequest(string $url): bool { return (bool)$found; } + /** + * @return void + */ + public static function waitForAsyncPropagation(): void { + usleep(self::asyncPropagationDelay() * 1000); + } + + /** + * wait for async propagation after a request + * if the request is a DAV request and is successful + * + * @param string $url + * @param string $method + * @param int $statusCode + * + * @return void + */ + public static function waitAsyncPropagationAfterRequest(string $url, string $method, int $statusCode): void { + $methods = ["POST", "PUT", "MOVE", "COPY", "DELETE"]; + + if (WebdavHelper::isDAVRequest($url) + && \str_starts_with($url, OcisHelper::getServerUrl()) + && \in_array($method, $methods) + && $statusCode < HttpResponse::HTTP_MULTIPLE_CHOICES + ) { + self::waitForAsyncPropagation(); + } + } + /** * clear space id reference for user * From 3ba79b18cfd213bcab01524c6b8e2725f573d09c Mon Sep 17 00:00:00 2001 From: Saw-jan Date: Tue, 26 Nov 2024 17:03:56 +0545 Subject: [PATCH 4/7] test: wait for MKCOL requests --- tests/acceptance/TestHelpers/WebDavHelper.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/acceptance/TestHelpers/WebDavHelper.php b/tests/acceptance/TestHelpers/WebDavHelper.php index ea85e0195a1..84af2471593 100644 --- a/tests/acceptance/TestHelpers/WebDavHelper.php +++ b/tests/acceptance/TestHelpers/WebDavHelper.php @@ -116,12 +116,12 @@ public static function waitForAsyncPropagation(): void { * @return void */ public static function waitAsyncPropagationAfterRequest(string $url, string $method, int $statusCode): void { - $methods = ["POST", "PUT", "MOVE", "COPY", "DELETE"]; + $methods = ["POST", "PUT", "MKCOL", "MOVE", "COPY", "DELETE"]; if (WebdavHelper::isDAVRequest($url) && \str_starts_with($url, OcisHelper::getServerUrl()) && \in_array($method, $methods) - && $statusCode < HttpResponse::HTTP_MULTIPLE_CHOICES + && $statusCode < 300 ) { self::waitForAsyncPropagation(); } From 68455899c13a6ba7e3b2192a0e3153c85bd89d7a Mon Sep 17 00:00:00 2001 From: Saw-jan Date: Tue, 26 Nov 2024 17:51:45 +0545 Subject: [PATCH 5/7] test: wait after share creation request --- tests/acceptance/TestHelpers/GraphHelper.php | 9 +++++++++ tests/acceptance/TestHelpers/WebDavHelper.php | 19 ++++++++++++++----- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/tests/acceptance/TestHelpers/GraphHelper.php b/tests/acceptance/TestHelpers/GraphHelper.php index c0f8df597db..0a4a7c7bab4 100644 --- a/tests/acceptance/TestHelpers/GraphHelper.php +++ b/tests/acceptance/TestHelpers/GraphHelper.php @@ -54,6 +54,15 @@ private static function getRequestHeaders(): array { ]; } + /** + * @param string $url + * + * @return bool + */ + public static function isShareRequest(string $url): bool { + return \str_ends_with($url, '/invite'); + } + /** * * @return string diff --git a/tests/acceptance/TestHelpers/WebDavHelper.php b/tests/acceptance/TestHelpers/WebDavHelper.php index 84af2471593..3e1ffddb796 100644 --- a/tests/acceptance/TestHelpers/WebDavHelper.php +++ b/tests/acceptance/TestHelpers/WebDavHelper.php @@ -101,8 +101,9 @@ public static function isDAVRequest(string $url): bool { /** * @return void */ - public static function waitForAsyncPropagation(): void { - usleep(self::asyncPropagationDelay() * 1000); + public static function waitForAsyncPropagation(int $extraWait): void { + $delay = self::asyncPropagationDelay() + $extraWait; + usleep($delay * 1000); } /** @@ -116,14 +117,22 @@ public static function waitForAsyncPropagation(): void { * @return void */ public static function waitAsyncPropagationAfterRequest(string $url, string $method, int $statusCode): void { - $methods = ["POST", "PUT", "MKCOL", "MOVE", "COPY", "DELETE"]; + $longerWaitMethods = ["MKCOL", "MOVE", "COPY", "DELETE"]; + $methods = \array_merge($longerWaitMethods, ["POST", "PUT"]); - if (WebdavHelper::isDAVRequest($url) + if ((WebdavHelper::isDAVRequest($url) || GraphHelper::isShareRequest($url)) && \str_starts_with($url, OcisHelper::getServerUrl()) && \in_array($method, $methods) && $statusCode < 300 ) { - self::waitForAsyncPropagation(); + $extraWait = 0; + if (\in_array($method, $longerWaitMethods)) { + // folder related requests require more time + // wait longer for these methods + // increase wait by 300ms + $extraWait = 300; + } + self::waitForAsyncPropagation($extraWait); } } From a4f80273c33eb24eeeebbf3b20db8b23bb34196c Mon Sep 17 00:00:00 2001 From: Saw-jan Date: Tue, 26 Nov 2024 17:53:32 +0545 Subject: [PATCH 6/7] test: fix php code docs --- tests/acceptance/TestHelpers/WebDavHelper.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/acceptance/TestHelpers/WebDavHelper.php b/tests/acceptance/TestHelpers/WebDavHelper.php index 3e1ffddb796..2142750ed88 100644 --- a/tests/acceptance/TestHelpers/WebDavHelper.php +++ b/tests/acceptance/TestHelpers/WebDavHelper.php @@ -99,6 +99,8 @@ public static function isDAVRequest(string $url): bool { } /** + * @param int $extraWait + * * @return void */ public static function waitForAsyncPropagation(int $extraWait): void { From 28f038409a22ca561f30546c309d6205e40022f6 Mon Sep 17 00:00:00 2001 From: Michael Barz Date: Wed, 27 Nov 2024 13:41:49 +0100 Subject: [PATCH 7/7] tests: enable async propagation for cs3apivalidator --- .drone.star | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.drone.star b/.drone.star index 238bac4fa7b..a629a29d290 100644 --- a/.drone.star +++ b/.drone.star @@ -23,7 +23,7 @@ OC_CI_GOLANG = "owncloudci/golang:1.22" OC_CI_NODEJS = "owncloudci/nodejs:%s" OC_CI_PHP = "owncloudci/php:%s" OC_CI_WAIT_FOR = "owncloudci/wait-for:latest" -OC_CS3_API_VALIDATOR = "owncloud/cs3api-validator:0.2.1" +OC_CS3_API_VALIDATOR = "owncloud/cs3api-validator:0.3.0" OC_LITMUS = "owncloudci/litmus:latest" OC_UBUNTU = "owncloud/ubuntu:20.04" ONLYOFFICE_DOCUMENT_SERVER = "onlyoffice/documentserver:7.5.1" @@ -1065,7 +1065,7 @@ def cs3ApiTests(ctx, storage, accounts_hash_difficulty = 4): "image": OC_CS3_API_VALIDATOR, "environment": {}, "commands": [ - "/usr/bin/cs3api-validator /var/lib/cs3api-validator --endpoint=%s:9142" % OCIS_SERVER_NAME, + "/usr/bin/cs3api-validator /var/lib/cs3api-validator --async-propagation=true --endpoint=%s:9142" % OCIS_SERVER_NAME, ], }, ],