diff --git a/tests/acceptance/bootstrap/CliContext.php b/tests/acceptance/bootstrap/CliContext.php index 0178e9a162..be6091f807 100644 --- a/tests/acceptance/bootstrap/CliContext.php +++ b/tests/acceptance/bootstrap/CliContext.php @@ -38,18 +38,27 @@ class CliContext implements Context { private SpacesContext $spacesContext; /** - * opencloud users storage path + * opencloud storage root path (with $HOME/~ expanded for CI) * * @return string */ - public static function getUsersStoragePath(): string { + public static function getStorageRootPath(): string { $path = getenv('OC_STORAGE_PATH') ?: '/var/lib/opencloud/storage/users'; // need for CI $home = getenv('HOME'); $path = preg_replace('#^~/#', $home . '/', $path); $path = str_replace('$HOME', $home, $path); - return rtrim($path, '/') . '/users'; + return rtrim($path, '/'); + } + + /** + * opencloud users storage path + * + * @return string + */ + public static function getUsersStoragePath(): string { + return self::getStorageRootPath() . '/users'; } /** @@ -58,8 +67,7 @@ public static function getUsersStoragePath(): string { * @return string */ public static function getProjectsStoragePath(): string { - $path = getenv('OC_STORAGE_PATH') ?: '/var/lib/opencloud/storage/users'; - return $path . '/projects'; + return self::getStorageRootPath() . '/projects'; } /** @@ -1039,4 +1047,261 @@ public function theAdminChecksTheAttributeOfFileForUser(string $attribute, strin ]; $this->featureContext->setResponse(CliHelper::runCommand($body)); } + + /** + * on-disk path of a project space root + * + * @param string $space + * + * @return string + */ + private function getProjectSpaceStoragePath(string $space): string { + $spaceId = $this->spacesContext->getSpaceIdByName($this->featureContext->getAdminUsername(), $space); + $spaceId = explode('$', $spaceId)[1]; + return $this->getProjectsStoragePath() . "/$spaceId"; + } + + /** + * + * @return void + */ + #[When('the administrator scans the whole storage using the CLI')] + public function theAdministratorScansTheWholeStorageUsingTheCli(): void { + $body = [ + "command" => "posixfs scan" + ]; + $this->featureContext->setResponse(CliHelper::runCommand($body)); + } + + /** + * + * @param string $folder + * @param string $user + * + * @return void + */ + #[When('the administrator scans the folder :folder of user :user using the CLI')] + public function theAdministratorScansTheFolderOfUserUsingTheCli(string $folder, string $user): void { + $userUuid = $this->featureContext->getAttributeOfCreatedUser($user, 'id'); + $storagePath = $this->getUsersStoragePath(); + $body = [ + "command" => "posixfs scan $storagePath/$userUuid/$folder" + ]; + $this->featureContext->setResponse(CliHelper::runCommand($body)); + } + + /** + * + * @param string $space + * + * @return void + */ + #[When('the administrator scans the space :space using the CLI')] + public function theAdministratorScansTheSpaceUsingTheCli(string $space): void { + $body = [ + "command" => "posixfs scan " . $this->getProjectSpaceStoragePath($space) + ]; + $this->featureContext->setResponse(CliHelper::runCommand($body)); + } + + /** + * + * @return void + */ + #[When('the administrator checks the posixfs consistency using the CLI')] + public function theAdministratorChecksThePosixfsConsistencyUsingTheCli(): void { + $body = [ + "command" => "posixfs consistency" + ]; + $this->featureContext->setResponse(CliHelper::runCommand($body)); + } + + /** + * + * @param string $flag + * + * @return void + */ + #[When('the administrator checks the posixfs consistency using the CLI with flag :flag')] + public function theAdministratorChecksThePosixfsConsistencyUsingTheCliWithFlag(string $flag): void { + $body = [ + "command" => "posixfs consistency $flag" + ]; + $this->featureContext->setResponse(CliHelper::runCommand($body)); + } + + /** + * + * @param string $file + * @param string $content + * @param string $space + * + * @return void + */ + #[When('the administrator creates the file :file with content :content in the space :space on the POSIX filesystem')] + public function theAdministratorCreatesFileInSpaceOnPosix(string $file, string $content, string $space): void { + $fullPath = $this->getProjectSpaceStoragePath($space) . "/$file"; + $safeContent = escapeshellarg($content); + $body = [ + "command" => "echo -n $safeContent > $fullPath", + "raw" => true + ]; + $this->featureContext->setResponse(CliHelper::runCommand($body)); + $this->waitForPath($fullPath); + } + + /** + * reads xattrs from disk with getfattr, bypassing the driver's on-the-fly assimilation + * + * @param string $file + * @param string $user + * + * @return void + */ + #[When('the administrator gets the extended attributes of file :file of user :user on the POSIX filesystem')] + public function theAdministratorGetsExtendedAttributesOfFileOfUser(string $file, string $user): void { + $userUuid = $this->featureContext->getAttributeOfCreatedUser($user, 'id'); + $storagePath = $this->getUsersStoragePath(); + $body = [ + "command" => "getfattr -d $storagePath/$userUuid/$file", + "raw" => true + ]; + $this->featureContext->setResponse(CliHelper::runCommand($body)); + } + + /** + * + * @param string $file + * @param string $space + * + * @return void + */ + #[When('the administrator gets the extended attributes of file :file in the space :space on the POSIX filesystem')] + public function theAdministratorGetsExtendedAttributesOfFileInSpace(string $file, string $space): void { + $body = [ + "command" => "getfattr -d " . $this->getProjectSpaceStoragePath($space) . "/$file", + "raw" => true + ]; + $this->featureContext->setResponse(CliHelper::runCommand($body)); + } + + /** + * + * @param string $attribute + * @param string $file + * @param string $user + * + * @return void + */ + #[When('the administrator gets the extended attribute :attribute of file :file of user :user on the POSIX filesystem')] + public function theAdministratorGetsExtendedAttributeOfFileOfUser(string $attribute, string $file, string $user): void { + $userUuid = $this->featureContext->getAttributeOfCreatedUser($user, 'id'); + $storagePath = $this->getUsersStoragePath(); + $body = [ + "command" => "getfattr -n " . escapeshellarg($attribute) . " --only-values $storagePath/$userUuid/$file", + "raw" => true + ]; + $this->featureContext->setResponse(CliHelper::runCommand($body)); + } + + /** + * + * @param string $attribute + * @param string $file + * @param string $user + * @param string $value + * + * @return void + */ + #[When('the administrator sets the extended attribute :attribute of file :file of user :user to :value on the POSIX filesystem')] + public function theAdministratorSetsExtendedAttributeOfFileOfUser( + string $attribute, + string $file, + string $user, + string $value + ): void { + $userUuid = $this->featureContext->getAttributeOfCreatedUser($user, 'id'); + $storagePath = $this->getUsersStoragePath(); + $body = [ + "command" => "setfattr -n " . escapeshellarg($attribute) + . " -v " . escapeshellarg($value) . " $storagePath/$userUuid/$file", + "raw" => true + ]; + $this->featureContext->setResponse(CliHelper::runCommand($body)); + } + + /** + * + * @param string $file + * @param string $user + * + * @return void + */ + #[When('the administrator scans the file :file of user :user using the CLI')] + public function theAdministratorScansTheFileOfUserUsingTheCli(string $file, string $user): void { + $userUuid = $this->featureContext->getAttributeOfCreatedUser($user, 'id'); + $storagePath = $this->getUsersStoragePath(); + $body = [ + "command" => "posixfs scan $storagePath/$userUuid/$file" + ]; + $this->featureContext->setResponse(CliHelper::runCommand($body)); + } + + /** + * + * @param string $path + * + * @return void + */ + #[When('the administrator scans path :path using the CLI')] + public function theAdministratorScansPathUsingTheCli(string $path): void { + $body = [ + "command" => "posixfs scan $path" + ]; + $this->featureContext->setResponse(CliHelper::runCommand($body)); + } + + /** + * non-existing path first, so with "-E" the scan aborts before the valid folder + * + * @param string $folder + * @param string $user + * @param string $flag + * + * @return void + */ + #[When('the administrator scans a non-existing path and the folder :folder of user :user using the CLI with flag :flag')] + public function theAdministratorScansNonExistingPathAndFolderUsingTheCli( + string $folder, + string $user, + string $flag + ): void { + $userUuid = $this->featureContext->getAttributeOfCreatedUser($user, 'id'); + $storagePath = $this->getUsersStoragePath(); + $nonExistingPath = "$storagePath/$userUuid/nonExistingPath"; + $validPath = "$storagePath/$userUuid/$folder"; + $flagPart = $flag !== "" ? "$flag " : ""; + $body = [ + "command" => "posixfs scan {$flagPart}$nonExistingPath $validPath" + ]; + $this->featureContext->setResponse(CliHelper::runCommand($body)); + } + + /** + * + * @return void + */ + #[Then('the command should not be successful')] + public function theCommandShouldNotBeSuccessful(): void { + $response = $this->featureContext->getResponse(); + $this->featureContext->theHTTPStatusCodeShouldBe(200, '', $response); + + $jsonResponse = $this->featureContext->getJsonDecodedResponse($response); + + Assert::assertNotSame( + 0, + $jsonResponse["exitCode"], + "Expected command to fail with a non-zero exit code, but got 0. Message: " . $jsonResponse["message"] + ); + } } diff --git a/tests/acceptance/features/cliCommands/posixfsConsistency.feature b/tests/acceptance/features/cliCommands/posixfsConsistency.feature new file mode 100644 index 0000000000..756e76b155 --- /dev/null +++ b/tests/acceptance/features/cliCommands/posixfsConsistency.feature @@ -0,0 +1,50 @@ +@env-config @skipOnOpencloud-decomposed-Storage @skipOnOpencloud-decomposeds3-Storage +Feature: posixfs consistency CLI command + As an administrator + I want to check and repair the consistency of the posix filesystem metadata + So that files with broken metadata (name, blobsize, parent id, ...) get fixed + + # WATCH_FS=false so the watcher does not re-assimilate the corrupted metadata before the command runs + Background: + Given the config "STORAGE_USERS_POSIX_WATCH_FS" has been set to "false" + And user "Alice" has been created with default attributes + + + Scenario: consistency fixes a corrupted name attribute + Given user "Alice" has uploaded file with content "content" to "textfile.txt" + When the administrator sets the extended attribute "user.oc.name" of file "textfile.txt" of user "Alice" to "corrupted.txt" on the POSIX filesystem + Then the command should be successful + When the administrator gets the extended attribute "user.oc.name" of file "textfile.txt" of user "Alice" on the POSIX filesystem + Then the command output should contain "corrupted.txt" + When the administrator checks the posixfs consistency using the CLI + Then the command should be successful + And the command output should contain "Fixed name attribute" + When the administrator gets the extended attribute "user.oc.name" of file "textfile.txt" of user "Alice" on the POSIX filesystem + Then the command output should contain "textfile.txt" + And the command output should not contain "corrupted.txt" + + + Scenario: consistency fixes a corrupted blobsize attribute + Given user "Alice" has uploaded file with content "content" to "textfile.txt" + When the administrator sets the extended attribute "user.oc.blobsize" of file "textfile.txt" of user "Alice" to "0" on the POSIX filesystem + Then the command should be successful + When the administrator checks the posixfs consistency using the CLI + Then the command should be successful + And the command output should contain "Fixed blobsize" + When the administrator gets the extended attribute "user.oc.blobsize" of file "textfile.txt" of user "Alice" on the POSIX filesystem + Then the command output should contain "7" + + + Scenario: consistency fixes corrupted checksums only with the --fix-checksums flag + Given user "Alice" has uploaded file with content "content" to "textfile.txt" + When the administrator sets the extended attribute "user.oc.cs.sha1" of file "textfile.txt" of user "Alice" to "corrupted" on the POSIX filesystem + Then the command should be successful + When the administrator checks the posixfs consistency using the CLI + Then the command should be successful + And the command output should not contain "Fixed checksum" + When the administrator checks the posixfs consistency using the CLI with flag "--fix-checksums" + Then the command should be successful + And the command output should contain "Fixed checksum" + When the administrator checks the posixfs consistency using the CLI with flag "--fix-checksums" + Then the command should be successful + And the command output should not contain "Fixed checksum" diff --git a/tests/acceptance/features/cliCommands/posixfsScan.feature b/tests/acceptance/features/cliCommands/posixfsScan.feature new file mode 100644 index 0000000000..f15febe1a4 --- /dev/null +++ b/tests/acceptance/features/cliCommands/posixfsScan.feature @@ -0,0 +1,104 @@ +@env-config @skipOnOpencloud-decomposed-Storage @skipOnOpencloud-decomposeds3-Storage +Feature: posixfs scan CLI command + As an administrator + I want to scan the posix filesystem for files that were added outside of OpenCloud + So that their metadata (ID cache and file metadata) gets assimilated and they become usable + + # WATCH_FS=false so only the scan command assimilates the files; proof read via getfattr from disk + Background: + Given the config "STORAGE_USERS_POSIX_WATCH_FS" has been set to "false" + And user "Alice" has been created with default attributes + + + Scenario: scan the whole storage assimilates files added directly to the personal and project spaces + Given the administrator has assigned the role "Space Admin" to user "Alice" using the Graph API + And user "Alice" has created a space "projectSpace1" with the default quota using the Graph API + And user "Alice" has created a space "projectSpace2" with the default quota using the Graph API + And the administrator has created the file "scanned.txt" with content "personal" for user "Alice" on the POSIX filesystem + And the administrator creates the file "inProject1.txt" with content "project one" in the space "projectSpace1" on the POSIX filesystem + And the administrator creates the file "inProject2.txt" with content "project two" in the space "projectSpace2" on the POSIX filesystem + When the administrator gets the extended attributes of file "scanned.txt" of user "Alice" on the POSIX filesystem + Then the command output should not contain "user.oc.id" + And the administrator gets the extended attributes of file "inProject1.txt" in the space "projectSpace1" on the POSIX filesystem + And the command output should not contain "user.oc.id" + And the administrator gets the extended attributes of file "inProject2.txt" in the space "projectSpace2" on the POSIX filesystem + And the command output should not contain "user.oc.id" + When the administrator scans the whole storage using the CLI + Then the command should be successful + And the command output should contain "Scan completed successfully." + When the administrator gets the extended attributes of file "scanned.txt" of user "Alice" on the POSIX filesystem + Then the command output should contain "user.oc.id" + When the administrator gets the extended attributes of file "inProject1.txt" in the space "projectSpace1" on the POSIX filesystem + Then the command output should contain "user.oc.id" + When the administrator gets the extended attributes of file "inProject2.txt" in the space "projectSpace2" on the POSIX filesystem + Then the command output should contain "user.oc.id" + And as "Alice" the final content of file "scanned.txt" should be "personal" + And using spaces DAV path + And for user "Alice" the space "projectSpace1" should contain these entries: + | inProject1.txt | + And for user "Alice" the space "projectSpace2" should contain these entries: + | inProject2.txt | + + + Scenario: scan a single folder assimilates the files inside it + Given the administrator creates the folder "scanFolder" for user "Alice" on the POSIX filesystem + And the administrator has created the file "scanFolder/inside.txt" with content "inside content" for user "Alice" on the POSIX filesystem + When the administrator gets the extended attributes of file "scanFolder/inside.txt" of user "Alice" on the POSIX filesystem + Then the command output should not contain "user.oc.id" + And the administrator scans the folder "scanFolder" of user "Alice" using the CLI + And the command should be successful + And the command output should contain "Scan completed successfully." + When the administrator gets the extended attributes of file "scanFolder/inside.txt" of user "Alice" on the POSIX filesystem + Then the command output should contain "user.oc.id" + And as "Alice" the final content of file "scanFolder/inside.txt" should be "inside content" + + + Scenario: scan a single project space assimilates only that space + Given the administrator has assigned the role "Space Admin" to user "Alice" using the Graph API + And user "Alice" has created a space "projectSpace1" with the default quota using the Graph API + And user "Alice" has created a space "projectSpace2" with the default quota using the Graph API + And the administrator creates the file "inProject1.txt" with content "project one" in the space "projectSpace1" on the POSIX filesystem + And the administrator creates the file "inProject2.txt" with content "project two" in the space "projectSpace2" on the POSIX filesystem + When the administrator scans the space "projectSpace1" using the CLI + Then the command should be successful + And the command output should contain "Scan completed successfully." + And the administrator gets the extended attributes of file "inProject1.txt" in the space "projectSpace1" on the POSIX filesystem + And the command output should contain "user.oc.id" + And the administrator gets the extended attributes of file "inProject2.txt" in the space "projectSpace2" on the POSIX filesystem + And the command output should not contain "user.oc.id" + And using spaces DAV path + And for user "Alice" the space "projectSpace1" should contain these entries: + | inProject1.txt | + + + Scenario: scan a single regular file assimilates only that file + Given the administrator has created the file "singleFile.txt" with content "single" for user "Alice" on the POSIX filesystem + And the administrator has created the file "otherFile.txt" with content "other" for user "Alice" on the POSIX filesystem + When the administrator scans the file "singleFile.txt" of user "Alice" using the CLI + Then the command should be successful + And the command output should contain "Scan completed successfully." + And the administrator gets the extended attributes of file "singleFile.txt" of user "Alice" on the POSIX filesystem + And the command output should contain "user.oc.id" + And the administrator gets the extended attributes of file "otherFile.txt" of user "Alice" on the POSIX filesystem + And the command output should not contain "user.oc.id" + And as "Alice" the final content of file "singleFile.txt" should be "single" + + + Scenario: scanning a path outside the storage root fails + When the administrator scans path "/tmp" using the CLI + Then the command should not be successful + And the command output should contain "does not appear to be inside a posixfs storage" + + + Scenario Outline: the halt-on-error flag controls whether the scan continues after an error + Given the administrator creates the folder "validFolder" for user "Alice" on the POSIX filesystem + And the administrator has created the file "validFolder/inside.txt" with content "keep going" for user "Alice" on the POSIX filesystem + When the administrator scans a non-existing path and the folder "validFolder" of user "Alice" using the CLI with flag "" + Then the command should not be successful + And the command output should contain "" + And the administrator gets the extended attributes of file "validFolder/inside.txt" of user "Alice" on the POSIX filesystem + And the command output contain "user.oc.id" + Examples: + | flag | message | shouldOrNot | + | | scan completed with 1 error | should | + | -E | scan aborted with 1 error | should not |