From ab2fbd79808d97d17dea068bf65bf020adb03b9f Mon Sep 17 00:00:00 2001 From: Christian Hartmann Date: Wed, 29 Jul 2026 22:25:49 +0200 Subject: [PATCH] feat: sync submissions to export file Signed-off-by: Christian Hartmann --- lib/Service/SubmissionService.php | 135 +++++++++++++++---- tests/Integration/Api/ApiV3Test.php | 125 +++++++++++++---- tests/Unit/Service/SubmissionServiceTest.php | 55 +++++--- 3 files changed, 241 insertions(+), 74 deletions(-) diff --git a/lib/Service/SubmissionService.php b/lib/Service/SubmissionService.php index 560da02fb..59799d06a 100644 --- a/lib/Service/SubmissionService.php +++ b/lib/Service/SubmissionService.php @@ -239,9 +239,10 @@ public function getSubmissionsData(Form $form, string $fileFormat, ?File $file = // Process initial header $header = []; - $header[] = $this->l10n->t('User ID'); - $header[] = $this->l10n->t('User display name'); - $header[] = $this->l10n->t('Timestamp'); + $header[] = ['id' => 'submission_id', 'title' => $this->l10n->t('Submission ID')]; + $header[] = ['id' => 'user_id', 'title' => $this->l10n->t('User ID')]; + $header[] = ['id' => 'user_display_name', 'title' => $this->l10n->t('User display name')]; + $header[] = ['id' => 'timestamp', 'title' => $this->l10n->t('Timestamp')]; /** @var array $questionPerQuestionId */ $questionPerQuestionId = []; /** @var array> $gridRowsPerQuestionId */ @@ -269,12 +270,12 @@ public function getSubmissionsData(Form $form, string $fileFormat, ?File $file = foreach ($gridRowsPerQuestionId[$question->getId()] as $rowId) { if ($gridCellType === Constants::ANSWER_GRID_TYPE_CHECKBOX || $gridCellType === Constants::ANSWER_GRID_TYPE_RADIO) { - $header[] = $question->getText() . ' (' . $optionPerOptionId[$rowId]->getText() . ')'; + $header[] = ['id' => 'question-id-' . $question->getId() . '-' . $rowId, 'title' => $question->getText() . ' (' . $optionPerOptionId[$rowId]->getText() . ')']; } if ($gridCellType === Constants::ANSWER_GRID_TYPE_NUMBER) { foreach ($gridColumnsPerQuestionId[$question->getId()] as $columnId) { - $header[] = $question->getText() . ' (' . $optionPerOptionId[$rowId]->getText() . ' - ' . $optionPerOptionId[$columnId]->getText() . ')'; + $header[] = ['id' => 'question-id-' . $question->getId() . '-' . $rowId . '-' . $columnId, 'title' => $question->getText() . ' (' . $optionPerOptionId[$rowId]->getText() . ' - ' . $optionPerOptionId[$columnId]->getText() . ')']; } } } @@ -285,10 +286,10 @@ public function getSubmissionsData(Form $form, string $fileFormat, ?File $file = $rankingOptionsPerQuestionId[$question->getId()][] = $option->getId(); } foreach ($rankingOptionsPerQuestionId[$question->getId()] as $optionId) { - $header[] = $question->getText() . ' (' . $optionPerOptionId[$optionId]->getText() . ')'; + $header[] = ['id' => 'question-id-' . $question->getId() . '-' . $optionId, 'title' => $question->getText() . ' (' . $optionPerOptionId[$optionId]->getText() . ')']; } } else { - $header[] = $question->getText(); + $header[] = ['id' => 'question-id-' . $question->getId(), 'title' => $question->getText()]; } $questionPerQuestionId[$question->getId()] = $question; @@ -301,6 +302,8 @@ public function getSubmissionsData(Form $form, string $fileFormat, ?File $file = foreach ($submissionEntities as $submission) { $row = []; + $row[] = $submission->getId(); + // User $user = $this->userManager->get($submission->getUserId()); if ($user === null) { @@ -388,7 +391,7 @@ function (array $carry, Answer $answer) use ($questionPerQuestionId, $gridRowsPe } /** - * @param array $header + * @param array $header * @param list, label?: string, url?: string}|mixed|null|string>> $data */ private function exportData(array $header, array $data, string $fileFormat, ?File $file = null): string { @@ -411,34 +414,120 @@ private function exportData(array $header, array $data, string $fileFormat, ?Fil } $activeWorksheet = $spreadsheet->getSheet(0); - foreach ($header as $columnIndex => $value) { - $activeWorksheet->setCellValue([$columnIndex + 1, 1], $value); + + // Set column IDs in a hidden row + $activeWorksheet->getRowDimension(2)->setVisible(false); + + // Get existing header + $existingHeaderIds = []; + $highestColumn = $activeWorksheet->getHighestColumn(); + $highestColumnIndex = \PhpOffice\PhpSpreadsheet\Cell\Coordinate::columnIndexFromString($highestColumn); + for ($col = 1; $col <= $highestColumnIndex; $col++) { + $id = $activeWorksheet->getCell([$col, 2])->getValue(); + if ($id) { + $existingHeaderIds[$id] = $col; + } + } + + $newHeaderIds = array_column($header, 'id'); + $newHeaderTitles = array_column($header, 'title'); + + // Sync Columns + $colsToDelete = array_diff(array_keys($existingHeaderIds), $newHeaderIds); + // Sort columns to delete by index descending + $colsToDeleteIndices = []; + foreach ($colsToDelete as $colId) { + if (isset($existingHeaderIds[$colId])) { + $colsToDeleteIndices[] = $existingHeaderIds[$colId]; + } + } + rsort($colsToDeleteIndices); + + foreach ($colsToDeleteIndices as $colIndex) { + $activeWorksheet->removeColumnByIndex($colIndex, 1); + } + + // Write header + foreach ($header as $columnIndex => $headerItem) { + $activeWorksheet->setCellValue([$columnIndex + 1, 2], $headerItem['id']); + $activeWorksheet->setCellValue([$columnIndex + 1, 1], $headerItem['title']); + } + + // Get existing submissions + $existingSubmissionIds = []; + $highestRow = $activeWorksheet->getHighestRow(); + $submissionIdColIndex = array_search('submission_id', $newHeaderIds); + $submissionIdCol = $submissionIdColIndex !== false ? $submissionIdColIndex + 1 : 0; + + if ($submissionIdCol) { + for ($row = 3; $row <= $highestRow; $row++) { + $submissionId = $activeWorksheet->getCell([$submissionIdCol, $row])->getValue(); + if ($submissionId) { + $existingSubmissionIds[(string)$submissionId] = $row; + } + } } - foreach ($data as $rowIndex => $rowData) { - $column = 1; - foreach ($rowData as $value) { - $row = $rowIndex + 2; + + $newSubmissionIds = []; + if ($submissionIdColIndex !== false) { + $newSubmissionIds = array_map('strval', array_column($data, $submissionIdColIndex)); + } + + // Sync Rows + $rowsToDelete = array_diff(array_keys($existingSubmissionIds), $newSubmissionIds); + $deletedCount = 0; + foreach ($rowsToDelete as $submissionId) { + $rowIndex = $existingSubmissionIds[$submissionId]; + $activeWorksheet->removeRow($rowIndex - $deletedCount, 1); + $deletedCount++; + } + + // Re-map existing submission rows after deletion + $existingSubmissionIds = []; + $highestRow = $activeWorksheet->getHighestRow(); + if ($submissionIdCol) { + for ($row = 3; $row <= $highestRow; $row++) { + $submissionId = $activeWorksheet->getCell([$submissionIdCol, $row])->getValue(); + if ($submissionId) { + $existingSubmissionIds[(string)$submissionId] = $row; + } + } + } + + // Update/Append data + foreach ($data as $rowData) { + $submissionId = (string)$rowData[$submissionIdColIndex]; + $row = $existingSubmissionIds[$submissionId] ?? null; + + if ($row === null) { + // Append new row + $row = $activeWorksheet->getHighestRow() + 1; + } + + $dataIndex = 0; + $columnIndex = 1; + while ($dataIndex < count($rowData)) { + $value = $rowData[$dataIndex]; if (is_array($value) && isset($value['label'])) { // file question type - $activeWorksheet->getCell([$column, $row]) + $activeWorksheet->getCell([$columnIndex, $row]) ->setValueExplicit($value['label']) ->getHyperlink() ->setUrl($value['url']); - - $activeWorksheet->getStyle([$column, $row]) + $activeWorksheet->getStyle([$columnIndex, $row]) ->getAlignment() ->setWrapText(true); + $columnIndex++; } elseif (is_array($value) && isset($value['columns'])) { // grid question type foreach ($value['columns'] as $nestedValue) { - $this->setCellValue($activeWorksheet, $column, $row, $nestedValue, $fileFormat); - $column++; + $this->setCellValue($activeWorksheet, $columnIndex, $row, $nestedValue, $fileFormat); + $columnIndex++; } - continue; // no need to increment the column one more time } else { - $this->setCellValue($activeWorksheet, $column, $row, $value, $fileFormat); + $this->setCellValue($activeWorksheet, $columnIndex, $row, $value, $fileFormat); + $columnIndex++; } - - $column++; + $dataIndex++; } } diff --git a/tests/Integration/Api/ApiV3Test.php b/tests/Integration/Api/ApiV3Test.php index e869016b2..aa8782b36 100644 --- a/tests/Integration/Api/ApiV3Test.php +++ b/tests/Integration/Api/ApiV3Test.php @@ -19,6 +19,8 @@ */ class ApiV3Test extends IntegrationBase { private Client $http; + private ?\OCP\Files\IRootFolder $rootFolder = null; + private ?\OCP\Files\Folder $userFolder = null; protected array $users = [ 'test' => 'Test user', @@ -267,6 +269,9 @@ public function setUp(): void { parent::setUp(); + $this->rootFolder = \OCP\Server::get(\OCP\Files\IRootFolder::class); + $this->userFolder = $this->rootFolder->getUserFolder('test'); + // Set up http Client $this->http = new Client([ 'base_uri' => 'http://localhost:8080/ocs/v2.php/apps/forms/', @@ -1337,35 +1342,6 @@ public function testGetSubmissions(array $expected) { $this->assertEquals($expected, $data); } - public static function dataExportSubmissions() { - return [ - 'exportSubmissions' => [ - 'expected' => <<<'CSV' - "User ID","User display name","Timestamp","First Question?","Second Question?","File Question?" - "","Anonymous user","1970-01-01T00:20:34+00:00","","","" - "","Anonymous user","1970-01-01T03:25:45+00:00","This is another short answer.","Option 2","" - "user1","User No. 1","1970-01-02T10:17:36+00:00","This is a short answer.","Option 1","" -CSV - ] - ]; - } - /** - * @dataProvider dataExportSubmissions - * - * @param array $expected - */ - public function testExportSubmissions(string $expected) { - $resp = $this->http->request('GET', "api/v3/forms/{$this->testForms[0]['id']}/submissions?fileFormat=csv"); - $data = substr((string)$resp->getBody()->getContents(), 3); // Some strange Character removed at the beginning - - $this->assertEquals(200, $resp->getStatusCode()); - $this->assertEquals('attachment; filename="Title of a Form (responses).csv"', $resp->getHeaders()['Content-Disposition'][0]); - $this->assertEquals('text/csv;charset=UTF-8', $resp->getHeaders()['Content-type'][0]); - $arr_txt_expected = preg_split('/,/', str_replace(["\t", "\n"], '', $expected)); - $arr_txt_data = preg_split('/,/', str_replace(["\t", "\n"], '', $data)); - $this->assertEquals($arr_txt_expected, $arr_txt_data); - } - public function testDeleteSubmission() { // Get submissions first to find a submission ID $resp = $this->http->request('GET', "api/v3/forms/{$this->testForms[0]['id']}/submissions"); @@ -1484,6 +1460,97 @@ public function testExportToCloud() { $this->assertEquals('Title of a Form (responses).csv', $data); } + public function testExportSubmissionsWithSync() { + $form = $this->testForms[0]; + $formId = $form['id']; + $question1 = $form['questions'][0]; + $question2 = $form['questions'][1]; + + // To be sure about IDs and order, let's fetch from API (newest first) + $resp = $this->http->request('GET', "api/v3/forms/{$formId}/submissions"); + $submissionsData = $this->OcsResponse2Data($resp); + $submissions = $submissionsData['submissions']; + $this->assertCount(3, $submissions, 'Pre-condition: Form should have 3 submissions'); + + $submission1_id = $submissions[0]['id']; // newest, user1 + $submission2_id = $submissions[1]['id']; // middle, user2 + $submission3_id = $submissions[2]['id']; // oldest, user3 + + // 1. Initial Export + $exportPath = '/Title of a Form (responses).csv'; + $this->http->request('POST', "api/v3/forms/{$formId}/submissions/export", ['json' => ['path' => '']]); + + // 2. Verify Initial Export + $content = $this->userFolder->get($exportPath)->getContent(); + $data = array_map('str_getcsv', explode("\n", trim($content))); + + $this->assertCount(5, $data, 'Expected 5 rows: header, hidden-header, 3 data rows'); + // Check headers + $this->assertStringContainsString('Submission ID', $data[0][0]); + $this->assertStringContainsString($question1['text'], $data[0][4]); + $this->assertStringContainsString($question2['text'], $data[0][5]); + + // Check hidden headers for IDs + $this->assertEquals('submission_id', $data[1][0], 'Hidden header for submission ID is missing or incorrect'); + $this->assertEquals('question-id-' . $question1['id'], $data[1][4]); + $this->assertEquals('question-id-' . $question2['id'], $data[1][5]); + + // Check data rows (oldest first: user3, user2, user1) + $this->assertEquals($submission3_id, $data[2][0]); + $this->assertEquals('', $data[2][4]); + $this->assertTrue(!isset($data[2][5]) || $data[2][5] === ''); + + $this->assertEquals($submission2_id, $data[3][0]); + $this->assertEquals('This is another short answer.', $data[3][4]); + $this->assertEquals('Option 2', $data[3][5]); + + $this->assertEquals($submission1_id, $data[4][0]); + $this->assertEquals('This is a short answer.', $data[4][4]); + $this->assertEquals('Option 1', $data[4][5]); + + // 3. Delete a submission and a question + $this->http->request('DELETE', "api/v3/forms/{$formId}/submissions/{$submission1_id}"); + $this->http->request('DELETE', "api/v3/forms/{$formId}/questions/{$question1['id']}"); + + // 4. Export again + $this->http->request('POST', "api/v3/forms/{$formId}/submissions/export", ['json' => ['path' => $exportPath, 'fileFormat' => 'csv']]); + + // 5. Verify Export after deletions + $content = $this->userFolder->get($exportPath)->getContent(); + $data = array_map('str_getcsv', explode("\n", trim($content))); + + $this->assertCount(4, $data, 'Expected 4 rows after deletion: header, hidden-header, 2 data rows'); + $this->assertStringNotContainsString($question1['text'], implode(',', $data[0])); + $this->assertStringContainsString($question2['text'], $data[0][4]); + + $this->assertEquals('question-id-' . $question2['id'], $data[1][4]); + // Check data rows (oldest first: user3, user2) + $this->assertEquals($submission3_id, $data[2][0]); + $this->assertTrue(!isset($data[2][4]) || $data[2][4] === ''); // No answer for q2 + $this->assertEquals($submission2_id, $data[3][0]); + $this->assertEquals('Option 2', $data[3][4]); + + // 6. Update a submission + $this->http->request('PUT', "api/v3/forms/{$formId}/submissions/{$submission2_id}", ['json' => ['answers' => [$question2['id'] => [$question2['options'][0]['id']]]]]); + + // 7. Export again + $this->http->request('POST', "api/v3/forms/{$formId}/submissions/export", ['json' => ['path' => $exportPath, 'fileFormat' => 'csv']]); + + // 8. Verify export after update + $content = $this->userFolder->get($exportPath)->getContent(); + $data = array_map('str_getcsv', explode("\n", trim($content))); + + $updatedRow = null; + foreach (array_slice($data, 2) as $row) { + if ($row[0] == $submission2_id) { + $updatedRow = $row; + break; + } + } + $this->assertNotNull($updatedRow, 'Updated submission not found in export'); + $this->assertEquals('Option 1', $updatedRow[4]); + } + public static function dataDeleteSubmissions() { $submissionsExpected = self::dataGetSubmissions()['getSubmissions']['expected']; $submissionsExpected['submissions'] = []; diff --git a/tests/Unit/Service/SubmissionServiceTest.php b/tests/Unit/Service/SubmissionServiceTest.php index d1400ec0c..211318079 100644 --- a/tests/Unit/Service/SubmissionServiceTest.php +++ b/tests/Unit/Service/SubmissionServiceTest.php @@ -382,9 +382,10 @@ public static function dataGetSubmissionsData() { ], // Expected CSV-Result ' - "User ID","User display name","Timestamp","Question 1","Question 2" - "user1","User 1","1973-11-29T22:33:09+01:00","Q1A2","Q2A2" - "user2","User 2","1973-11-29T22:33:09+01:00","Q1A1","Q2A1" + "Submission ID","User ID","User display name","Timestamp","Question 1","Question 2" + "submission_id","user_id","user_display_name","timestamp","question-id-1","question-id-2" + "2","user1","User 1","1973-11-29T22:33:09+01:00","Q1A2","Q2A2" + "1","user2","User 2","1973-11-29T22:33:09+01:00","Q1A1","Q2A1" ' ], 'checkbox-multi-answers' => [ @@ -407,8 +408,9 @@ public static function dataGetSubmissionsData() { ], // Expected CSV-Result ' - "User ID","User display name","Timestamp","Question 1" - "user1","User 1","1973-11-29T22:33:09+01:00","Q1A1; Q1A2; Q1A3" + "Submission ID","User ID","User display name","Timestamp","Question 1" + "submission_id","user_id","user_display_name","timestamp","question-id-1" + "1","user1","User 1","1973-11-29T22:33:09+01:00","Q1A1; Q1A2; Q1A3" ' ], 'grid-checkbox-multi-answers' => [ @@ -434,8 +436,9 @@ public static function dataGetSubmissionsData() { ], // Expected CSV-Result ' - "User ID","User display name","Timestamp","Question 1 (Option 2)","Question 1 (Option 3)" - "user1","User 1","1973-11-29T22:33:09+01:00","Option 55","Option 56" + "Submission ID","User ID","User display name","Timestamp","Question 1 (Option 2)","Question 1 (Option 3)" + "submission_id","user_id","user_display_name","timestamp","question-id-1-2","question-id-1-3" + "1","user1","User 1","1973-11-29T22:33:09+01:00","Option 55","Option 56" ' ], 'file-multi-answers' => [ @@ -457,8 +460,9 @@ public static function dataGetSubmissionsData() { ], // Expected CSV-Result ' - "User ID","User display name","Timestamp","Question 1" - "user1","User 1","1973-11-29T22:33:09+01:00","file1.txt; ' . ' + "Submission ID","User ID","User display name","Timestamp","Question 1" + "submission_id","user_id","user_display_name","timestamp","question-id-1" + "1","user1","User 1","1973-11-29T22:33:09+01:00","file1.txt; ' . ' file2.txt" ' ], @@ -480,8 +484,9 @@ public static function dataGetSubmissionsData() { ], // Expected CSV-Result ' - "User ID","User display name","Timestamp","Question 1" - "","Anonymous user","1973-11-29T22:33:09+01:00","Q1A1" + "Submission ID","User ID","User display name","Timestamp","Question 1" + "submission_id","user_id","user_display_name","timestamp","question-id-1" + "1","","Anonymous user","1973-11-29T22:33:09+01:00","Q1A1" ' ], 'questions-not-answered' => [ @@ -504,8 +509,9 @@ public static function dataGetSubmissionsData() { ], // Expected CSV-Result ' - "User ID","User display name","Timestamp","Question 1","Question 2","Question 3" - "user1","User 1","1973-11-29T22:33:09+01:00","","Q2A1","" + "Submission ID","User ID","User display name","Timestamp","Question 1","Question 2","Question 3" + "submission_id","user_id","user_display_name","timestamp","question-id-1","question-id-2","question-id-3" + "1","user1","User 1","1973-11-29T22:33:09+01:00","","Q2A1","" ' ], /* No submissions, but request via api */ @@ -518,7 +524,8 @@ public static function dataGetSubmissionsData() { [], // Expected CSV-Result ' - "User ID","User display name","Timestamp","Question 1" + "Submission ID","User ID","User display name","Timestamp","Question 1" + "submission_id","user_id","user_display_name","timestamp","question-id-1" ' ], /* All Questions e.g. got deleted */ @@ -538,8 +545,9 @@ public static function dataGetSubmissionsData() { ], // Expected CSV-Result ' - "User ID","User display name","Timestamp" - "","Anonymous user","1973-11-29T22:33:09+01:00" + "Submission ID","User ID","User display name","Timestamp" + "submission_id","user_id","user_display_name","timestamp" + "1","","Anonymous user","1973-11-29T22:33:09+01:00" ' ], 'ranking-submission' => [ @@ -564,8 +572,9 @@ public static function dataGetSubmissionsData() { ], // Expected CSV-Result: one column per option with rank position ' - "User ID","User display name","Timestamp","Rank these (Option A)","Rank these (Option B)","Rank these (Option C)" - "user1","User 1","1973-11-29T22:33:09+01:00","2","1","3" + "Submission ID","User ID","User display name","Timestamp","Rank these (Option A)","Rank these (Option B)","Rank these (Option C)" + "submission_id","user_id","user_display_name","timestamp","question-id-1-10","question-id-1-11","question-id-1-12" + "1","user1","User 1","1973-11-29T22:33:09+01:00","2","1","3" ' ], 'ranking-unanswered' => [ @@ -587,8 +596,9 @@ public static function dataGetSubmissionsData() { ], // Expected CSV-Result: columns exist but values are empty ' - "User ID","User display name","Timestamp","Rank these (Option A)","Rank these (Option B)" - "user1","User 1","1973-11-29T22:33:09+01:00","","" + "Submission ID","User ID","User display name","Timestamp","Rank these (Option A)","Rank these (Option B)" + "submission_id","user_id","user_display_name","timestamp","question-id-1-10","question-id-1-11" + "1","user1","User 1","1973-11-29T22:33:09+01:00","","" ' ], ]; @@ -687,8 +697,9 @@ private function setUpSimpleCsvTest(string $formTitle = 'Some nice Form Title'): ], // Expected CSV-Result ' - "User ID","User display name","Timestamp","Question 1" - "user1","User 1","1973-11-29T22:33:09+01:00","Q1A1" + "Submission ID","User ID","User display name","Timestamp","Question 1" + "submission_id","user_id","user_display_name","timestamp","question-id-1" + "1","user1","User 1","1973-11-29T22:33:09+01:00","Q1A1" ', // Form title $formTitle