Skip to content

feat: sync submissions to export file #3546

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Chartman123 wants to merge 1 commit into
base: main
Choose a base branch
from
+241 −74
Draft

feat: sync submissions to export file #3546

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
135 changes: 112 additions & 23 deletions lib/Service/SubmissionService.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -239,9 +239,10 @@ public function getSubmissionsData(Form $form, string $fileFormat, ?File $file =

// Process initial header
$header = [];
$header[] = $this->l10n->t('User ID');
$header[] = $this->l10n->t('User display name');
$header[] = $this->l10n->t('Timestamp');
$header[] = ['id' => 'submission_id', 'title' => $this->l10n->t('Submission ID')];
$header[] = ['id' => 'user_id', 'title' => $this->l10n->t('User ID')];
$header[] = ['id' => 'user_display_name', 'title' => $this->l10n->t('User display name')];
$header[] = ['id' => 'timestamp', 'title' => $this->l10n->t('Timestamp')];
/** @var array<int, Question> $questionPerQuestionId */
$questionPerQuestionId = [];
/** @var array<int, array<int, string>> $gridRowsPerQuestionId */
Expand Down Expand Up @@ -269,12 +270,12 @@ public function getSubmissionsData(Form $form, string $fileFormat, ?File $file =

foreach ($gridRowsPerQuestionId[$question->getId()] as $rowId) {
if ($gridCellType === Constants::ANSWER_GRID_TYPE_CHECKBOX || $gridCellType === Constants::ANSWER_GRID_TYPE_RADIO) {
$header[] = $question->getText() . ' (' . $optionPerOptionId[$rowId]->getText() . ')';
$header[] = ['id' => 'question-id-' . $question->getId() . '-' . $rowId, 'title' => $question->getText() . ' (' . $optionPerOptionId[$rowId]->getText() . ')'];
}

if ($gridCellType === Constants::ANSWER_GRID_TYPE_NUMBER) {
foreach ($gridColumnsPerQuestionId[$question->getId()] as $columnId) {
$header[] = $question->getText() . ' (' . $optionPerOptionId[$rowId]->getText() . ' - ' . $optionPerOptionId[$columnId]->getText() . ')';
$header[] = ['id' => 'question-id-' . $question->getId() . '-' . $rowId . '-' . $columnId, 'title' => $question->getText() . ' (' . $optionPerOptionId[$rowId]->getText() . ' - ' . $optionPerOptionId[$columnId]->getText() . ')'];
}
}
}
Expand All @@ -285,10 +286,10 @@ public function getSubmissionsData(Form $form, string $fileFormat, ?File $file =
$rankingOptionsPerQuestionId[$question->getId()][] = $option->getId();
}
foreach ($rankingOptionsPerQuestionId[$question->getId()] as $optionId) {
$header[] = $question->getText() . ' (' . $optionPerOptionId[$optionId]->getText() . ')';
$header[] = ['id' => 'question-id-' . $question->getId() . '-' . $optionId, 'title' => $question->getText() . ' (' . $optionPerOptionId[$optionId]->getText() . ')'];
}
} else {
$header[] = $question->getText();
$header[] = ['id' => 'question-id-' . $question->getId(), 'title' => $question->getText()];
}

$questionPerQuestionId[$question->getId()] = $question;
Expand All @@ -301,6 +302,8 @@ public function getSubmissionsData(Form $form, string $fileFormat, ?File $file =
foreach ($submissionEntities as $submission) {
$row = [];

$row[] = $submission->getId();

// User
$user = $this->userManager->get($submission->getUserId());
if ($user === null) {
Expand Down Expand Up @@ -388,7 +391,7 @@ function (array $carry, Answer $answer) use ($questionPerQuestionId, $gridRowsPe
}

/**
* @param array<int, string> $header
* @param array<int, array{id: string, title: string}> $header
* @param list<non-empty-list<array{columns?: list<mixed|string>, label?: string, url?: string}|mixed|null|string>> $data
*/
private function exportData(array $header, array $data, string $fileFormat, ?File $file = null): string {
Expand All @@ -411,34 +414,120 @@ private function exportData(array $header, array $data, string $fileFormat, ?Fil
}

$activeWorksheet = $spreadsheet->getSheet(0);
foreach ($header as $columnIndex => $value) {
$activeWorksheet->setCellValue([$columnIndex + 1, 1], $value);

// Set column IDs in a hidden row
$activeWorksheet->getRowDimension(2)->setVisible(false);

// Get existing header
$existingHeaderIds = [];
$highestColumn = $activeWorksheet->getHighestColumn();
$highestColumnIndex = \PhpOffice\PhpSpreadsheet\Cell\Coordinate::columnIndexFromString($highestColumn);
for ($col = 1; $col <= $highestColumnIndex; $col++) {
$id = $activeWorksheet->getCell([$col, 2])->getValue();
if ($id) {
$existingHeaderIds[$id] = $col;
}
}

$newHeaderIds = array_column($header, 'id');
$newHeaderTitles = array_column($header, 'title');

// Sync Columns
$colsToDelete = array_diff(array_keys($existingHeaderIds), $newHeaderIds);
// Sort columns to delete by index descending
$colsToDeleteIndices = [];
foreach ($colsToDelete as $colId) {
if (isset($existingHeaderIds[$colId])) {
$colsToDeleteIndices[] = $existingHeaderIds[$colId];
}
}
rsort($colsToDeleteIndices);

foreach ($colsToDeleteIndices as $colIndex) {
$activeWorksheet->removeColumnByIndex($colIndex, 1);
}

// Write header
foreach ($header as $columnIndex => $headerItem) {
$activeWorksheet->setCellValue([$columnIndex + 1, 2], $headerItem['id']);
$activeWorksheet->setCellValue([$columnIndex + 1, 1], $headerItem['title']);
}

// Get existing submissions
$existingSubmissionIds = [];
$highestRow = $activeWorksheet->getHighestRow();
$submissionIdColIndex = array_search('submission_id', $newHeaderIds);
$submissionIdCol = $submissionIdColIndex !== false ? $submissionIdColIndex + 1 : 0;

if ($submissionIdCol) {
for ($row = 3; $row <= $highestRow; $row++) {
$submissionId = $activeWorksheet->getCell([$submissionIdCol, $row])->getValue();
if ($submissionId) {
$existingSubmissionIds[(string)$submissionId] = $row;
}
}
}
foreach ($data as $rowIndex => $rowData) {
$column = 1;
foreach ($rowData as $value) {
$row = $rowIndex + 2;

$newSubmissionIds = [];
if ($submissionIdColIndex !== false) {
$newSubmissionIds = array_map('strval', array_column($data, $submissionIdColIndex));
}

// Sync Rows
$rowsToDelete = array_diff(array_keys($existingSubmissionIds), $newSubmissionIds);
$deletedCount = 0;
foreach ($rowsToDelete as $submissionId) {
$rowIndex = $existingSubmissionIds[$submissionId];
$activeWorksheet->removeRow($rowIndex - $deletedCount, 1);
$deletedCount++;
}

// Re-map existing submission rows after deletion
$existingSubmissionIds = [];
$highestRow = $activeWorksheet->getHighestRow();
if ($submissionIdCol) {
for ($row = 3; $row <= $highestRow; $row++) {
$submissionId = $activeWorksheet->getCell([$submissionIdCol, $row])->getValue();
if ($submissionId) {
$existingSubmissionIds[(string)$submissionId] = $row;
}
}
}

// Update/Append data
foreach ($data as $rowData) {
$submissionId = (string)$rowData[$submissionIdColIndex];
$row = $existingSubmissionIds[$submissionId] ?? null;

if ($row === null) {
// Append new row
$row = $activeWorksheet->getHighestRow() + 1;
}

$dataIndex = 0;
$columnIndex = 1;
while ($dataIndex < count($rowData)) {
$value = $rowData[$dataIndex];

if (is_array($value) && isset($value['label'])) { // file question type
$activeWorksheet->getCell([$column, $row])
$activeWorksheet->getCell([$columnIndex, $row])
->setValueExplicit($value['label'])
->getHyperlink()
->setUrl($value['url']);

$activeWorksheet->getStyle([$column, $row])
$activeWorksheet->getStyle([$columnIndex, $row])
->getAlignment()
->setWrapText(true);
$columnIndex++;
} elseif (is_array($value) && isset($value['columns'])) { // grid question type
foreach ($value['columns'] as $nestedValue) {
$this->setCellValue($activeWorksheet, $column, $row, $nestedValue, $fileFormat);
$column++;
$this->setCellValue($activeWorksheet, $columnIndex, $row, $nestedValue, $fileFormat);
$columnIndex++;
}
continue; // no need to increment the column one more time
} else {
$this->setCellValue($activeWorksheet, $column, $row, $value, $fileFormat);
$this->setCellValue($activeWorksheet, $columnIndex, $row, $value, $fileFormat);
$columnIndex++;
}

$column++;
$dataIndex++;
}
}

Expand Down
125 changes: 96 additions & 29 deletions tests/Integration/Api/ApiV3Test.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@
*/
class ApiV3Test extends IntegrationBase {
private Client $http;
private ?\OCP\Files\IRootFolder $rootFolder = null;
private ?\OCP\Files\Folder $userFolder = null;

protected array $users = [
'test' => 'Test user',
Expand Down Expand Up @@ -267,6 +269,9 @@ public function setUp(): void {

parent::setUp();

$this->rootFolder = \OCP\Server::get(\OCP\Files\IRootFolder::class);
$this->userFolder = $this->rootFolder->getUserFolder('test');

// Set up http Client
$this->http = new Client([
'base_uri' => 'http://localhost:8080/ocs/v2.php/apps/forms/',
Expand Down Expand Up @@ -1337,35 +1342,6 @@ public function testGetSubmissions(array $expected) {
$this->assertEquals($expected, $data);
}

public static function dataExportSubmissions() {
return [
'exportSubmissions' => [
'expected' => <<<'CSV'
"User ID","User display name","Timestamp","First Question?","Second Question?","File Question?"
"","Anonymous user","1970-01-01T00:20:34+00:00","","",""
"","Anonymous user","1970-01-01T03:25:45+00:00","This is another short answer.","Option 2",""
"user1","User No. 1","1970-01-02T10:17:36+00:00","This is a short answer.","Option 1",""
CSV
]
];
}
/**
* @dataProvider dataExportSubmissions
*
* @param array $expected
*/
public function testExportSubmissions(string $expected) {
$resp = $this->http->request('GET', "api/v3/forms/{$this->testForms[0]['id']}/submissions?fileFormat=csv");
$data = substr((string)$resp->getBody()->getContents(), 3); // Some strange Character removed at the beginning

$this->assertEquals(200, $resp->getStatusCode());
$this->assertEquals('attachment; filename="Title of a Form (responses).csv"', $resp->getHeaders()['Content-Disposition'][0]);
$this->assertEquals('text/csv;charset=UTF-8', $resp->getHeaders()['Content-type'][0]);
$arr_txt_expected = preg_split('/,/', str_replace(["\t", "\n"], '', $expected));
$arr_txt_data = preg_split('/,/', str_replace(["\t", "\n"], '', $data));
$this->assertEquals($arr_txt_expected, $arr_txt_data);
}

public function testDeleteSubmission() {
// Get submissions first to find a submission ID
$resp = $this->http->request('GET', "api/v3/forms/{$this->testForms[0]['id']}/submissions");
Expand Down Expand Up @@ -1484,6 +1460,97 @@ public function testExportToCloud() {
$this->assertEquals('Title of a Form (responses).csv', $data);
}

public function testExportSubmissionsWithSync() {
$form = $this->testForms[0];
$formId = $form['id'];
$question1 = $form['questions'][0];
$question2 = $form['questions'][1];

// To be sure about IDs and order, let's fetch from API (newest first)
$resp = $this->http->request('GET', "api/v3/forms/{$formId}/submissions");
$submissionsData = $this->OcsResponse2Data($resp);
$submissions = $submissionsData['submissions'];
$this->assertCount(3, $submissions, 'Pre-condition: Form should have 3 submissions');

$submission1_id = $submissions[0]['id']; // newest, user1
$submission2_id = $submissions[1]['id']; // middle, user2
$submission3_id = $submissions[2]['id']; // oldest, user3

// 1. Initial Export
$exportPath = '/Title of a Form (responses).csv';
$this->http->request('POST', "api/v3/forms/{$formId}/submissions/export", ['json' => ['path' => '']]);

// 2. Verify Initial Export
$content = $this->userFolder->get($exportPath)->getContent();
$data = array_map('str_getcsv', explode("\n", trim($content)));

$this->assertCount(5, $data, 'Expected 5 rows: header, hidden-header, 3 data rows');
// Check headers
$this->assertStringContainsString('Submission ID', $data[0][0]);
$this->assertStringContainsString($question1['text'], $data[0][4]);
$this->assertStringContainsString($question2['text'], $data[0][5]);

// Check hidden headers for IDs
$this->assertEquals('submission_id', $data[1][0], 'Hidden header for submission ID is missing or incorrect');
$this->assertEquals('question-id-' . $question1['id'], $data[1][4]);
$this->assertEquals('question-id-' . $question2['id'], $data[1][5]);

// Check data rows (oldest first: user3, user2, user1)
$this->assertEquals($submission3_id, $data[2][0]);
$this->assertEquals('', $data[2][4]);
$this->assertTrue(!isset($data[2][5]) || $data[2][5] === '');

$this->assertEquals($submission2_id, $data[3][0]);
$this->assertEquals('This is another short answer.', $data[3][4]);
$this->assertEquals('Option 2', $data[3][5]);

$this->assertEquals($submission1_id, $data[4][0]);
$this->assertEquals('This is a short answer.', $data[4][4]);
$this->assertEquals('Option 1', $data[4][5]);

// 3. Delete a submission and a question
$this->http->request('DELETE', "api/v3/forms/{$formId}/submissions/{$submission1_id}");
$this->http->request('DELETE', "api/v3/forms/{$formId}/questions/{$question1['id']}");

// 4. Export again
$this->http->request('POST', "api/v3/forms/{$formId}/submissions/export", ['json' => ['path' => $exportPath, 'fileFormat' => 'csv']]);

// 5. Verify Export after deletions
$content = $this->userFolder->get($exportPath)->getContent();
$data = array_map('str_getcsv', explode("\n", trim($content)));

$this->assertCount(4, $data, 'Expected 4 rows after deletion: header, hidden-header, 2 data rows');
$this->assertStringNotContainsString($question1['text'], implode(',', $data[0]));
$this->assertStringContainsString($question2['text'], $data[0][4]);

$this->assertEquals('question-id-' . $question2['id'], $data[1][4]);
// Check data rows (oldest first: user3, user2)
$this->assertEquals($submission3_id, $data[2][0]);
$this->assertTrue(!isset($data[2][4]) || $data[2][4] === ''); // No answer for q2
$this->assertEquals($submission2_id, $data[3][0]);
$this->assertEquals('Option 2', $data[3][4]);

// 6. Update a submission
$this->http->request('PUT', "api/v3/forms/{$formId}/submissions/{$submission2_id}", ['json' => ['answers' => [$question2['id'] => [$question2['options'][0]['id']]]]]);

// 7. Export again
$this->http->request('POST', "api/v3/forms/{$formId}/submissions/export", ['json' => ['path' => $exportPath, 'fileFormat' => 'csv']]);

// 8. Verify export after update
$content = $this->userFolder->get($exportPath)->getContent();
$data = array_map('str_getcsv', explode("\n", trim($content)));

$updatedRow = null;
foreach (array_slice($data, 2) as $row) {
if ($row[0] == $submission2_id) {
$updatedRow = $row;
break;
}
}
$this->assertNotNull($updatedRow, 'Updated submission not found in export');
$this->assertEquals('Option 1', $updatedRow[4]);
}

public static function dataDeleteSubmissions() {
$submissionsExpected = self::dataGetSubmissions()['getSubmissions']['expected'];
$submissionsExpected['submissions'] = [];
Expand Down
Loading
Loading