UNOMI-998: Cover a 3.0 to 4.0 migration in the integration tests

itests/src/test/java/org/apache/unomi/itests/AllITs.java

@@ -18,6 +18,7 @@
 
 import org.apache.unomi.itests.migration.Migrate16xToCurrentVersionIT;
 import org.apache.unomi.itests.graphql.*;
+import org.apache.unomi.itests.migration.Migrate30ToCurrentVersionIT;
 import org.apache.unomi.itests.migration.Migrate40IndexMappingIT;
 import org.apache.unomi.itests.migration.Migrate40TenantAssignmentIT;
 import org.apache.unomi.itests.migration.MigrationIT;
@@ -34,6 +35,7 @@
  */
 @RunWith(ProgressSuite.class)
 @SuiteClasses({
+        Migrate30ToCurrentVersionIT.class,
         Migrate16xToCurrentVersionIT.class,
         MigrationIT.class,
         Migrate40IndexMappingIT.class,

itests/src/test/java/org/apache/unomi/itests/CorePersistenceITs.java

@@ -18,6 +18,7 @@
 
 import org.apache.unomi.itests.migration.Migrate16xToCurrentVersionIT;
 import org.apache.unomi.itests.graphql.*;
+import org.apache.unomi.itests.migration.Migrate30ToCurrentVersionIT;
 import org.apache.unomi.itests.migration.Migrate40IndexMappingIT;
 import org.apache.unomi.itests.migration.Migrate40TenantAssignmentIT;
 import org.apache.unomi.itests.migration.MigrationIT;
@@ -35,6 +36,7 @@
  */
 @RunWith(ProgressSuite.class)
 @SuiteClasses({
+        Migrate30ToCurrentVersionIT.class,
         Migrate16xToCurrentVersionIT.class,
         MigrationIT.class,
         Migrate40IndexMappingIT.class,

itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xToCurrentVersionIT.java

@@ -40,6 +40,8 @@
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.util.*;
 
 @Category(SearchBackendIT.class)
@@ -125,6 +127,15 @@ public void waitForStartup() throws InterruptedException {
 
         // Restore snapshot from 1.6.x
         try (CloseableHttpClient httpClient = createSearchEngineHttpClient()) {
+            // A prior migration IT (e.g. Migrate30) may have left open context-* indices.
+            // Snapshot restore refuses to overwrite open indices, so clear them first.
+            for (String index : MigrationUtils.getIndexesPrefixedBy(httpClient, getEsBaseUrl(), INDEX_PREFIX_CONTEXT)) {
+                LOGGER.info("Deleting existing index before 1.6 snapshot restore: {}", index);
+                MigrationUtils.deleteIndex(httpClient, getEsBaseUrl(), index);
+            }
+            // Its completed history would make this migration skip every 4.0.0 step.
+            Files.deleteIfExists(Paths.get(karafData(), "migration", "history.json"));
+
             // Create snapshot repo
             HttpUtils.executePutRequest(httpClient, getEsSnapshotRepo(), resourceAsString(RESOURCE_CREATE_SNAPSHOTS_REPO), null);
             // Get snapshot, insure it exists

itests/src/test/java/org/apache/unomi/itests/persistence/OpenSearchITBackend.java

@@ -82,15 +82,25 @@ public Option[] featureOptions() {
     @Override
     public Option[] configurationOptions() {
         String port = searchPort();
+        String address = "localhost:" + port;
         return new Option[]{
                 editConfigurationFilePut("etc/custom.system.properties", "org.apache.unomi.opensearch.cluster.name", "contextElasticSearchITests"),
-                editConfigurationFilePut("etc/custom.system.properties", "org.apache.unomi.opensearch.addresses", "localhost:" + port),
+                editConfigurationFilePut("etc/custom.system.properties", "org.apache.unomi.opensearch.addresses", address),
                 editConfigurationFilePut("etc/custom.system.properties", "org.apache.unomi.opensearch.username", OS_USER),
                 editConfigurationFilePut("etc/custom.system.properties", "org.apache.unomi.opensearch.password", OS_PASSWORD),
                 editConfigurationFilePut("etc/custom.system.properties", "org.apache.unomi.opensearch.sslEnable", "false"),
                 editConfigurationFilePut("etc/custom.system.properties", "org.apache.unomi.opensearch.sslTrustAllCertificates", "true"),
                 editConfigurationFilePut("etc/custom.system.properties", "org.apache.unomi.opensearch.rollover.maxDocs", "300"),
                 editConfigurationFilePut("etc/custom.system.properties", "org.apache.unomi.opensearch.minimalClusterState", "YELLOW"),
+                // org.apache.unomi.migration.cfg still resolves connection settings from the
+                // elasticsearch.* placeholders. Mirror OpenSearch here so unomi:migrate (e.g.
+                // Migrate30ToCurrentVersionIT) does not fall back to localhost:9200.
+                editConfigurationFilePut("etc/custom.system.properties", "org.apache.unomi.elasticsearch.addresses", address),
+                editConfigurationFilePut("etc/custom.system.properties", "org.apache.unomi.elasticsearch.username", OS_USER),
+                editConfigurationFilePut("etc/custom.system.properties", "org.apache.unomi.elasticsearch.password", OS_PASSWORD),
+                editConfigurationFilePut("etc/custom.system.properties", "org.apache.unomi.elasticsearch.sslEnable", "false"),
+                editConfigurationFilePut("etc/custom.system.properties", "org.apache.unomi.elasticsearch.sslTrustAllCertificates", "true"),
+                editConfigurationFilePut("etc/custom.system.properties", "org.apache.unomi.elasticsearch.rollover.maxDocs", "300"),
         };
     }
 

tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-4.0.0-10-tenantInitialization.groovy

@@ -107,10 +107,10 @@ ${sep}
 Generated : ${isoDate}
 Tenant ID : ${tenantId}
 
-PUBLIC KEY  (X-Unomi-Public-Key header -- context.json / event collector):
+PUBLIC KEY  (X-Unomi-Api-Key header -- context.json / event collector):
   ${generatedPublicKey}
 
-PRIVATE KEY (X-Unomi-Key header -- protected / admin endpoints):
+PRIVATE KEY  (HTTP Basic authentication as tenantId:privateKey -- protected / admin endpoints):
   ${generatedPrivateKey}
 
 IMPORTANT: These keys cannot be recovered after this file is deleted.

tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg

@@ -17,7 +17,9 @@
 
 # Migration config used for silent migration
 
-# Various configuration related to ElasticSearch communication to be able to connect and migrate the data
+# Various configuration related to search-engine communication to be able to connect and migrate the data.
+# These keys read the org.apache.unomi.elasticsearch.* system properties for historical reasons, also when
+# the search engine is OpenSearch: set them to your OpenSearch connection settings before running unomi:migrate.
 esAddresses = ${org.apache.unomi.elasticsearch.addresses:-localhost:9200}
 esSSLEnabled = ${org.apache.unomi.elasticsearch.sslEnable:-false}
 esLogin = ${org.apache.unomi.elasticsearch.username:-}

tools/shell-commands/src/main/resources/requestBody/4.0.0/initialize_tenant_and_audit_fields.painless

@@ -20,12 +20,18 @@
 // stay in the configured tenant even when their 3.0 scope is "systemscope".
 // A shipped definition is recognised by item type and id only, so a 3.0 item sharing both
 // (such as a built-in rule edited in place) also goes to system.
+// The reindex reads from the "-cloned" backup and ctx._index is that source index, so drop the suffix.
+def indexLower = ctx._index != null ? ctx._index.toLowerCase(Locale.ROOT) : '';
+if (indexLower.endsWith('-cloned')) {
+    indexLower = indexLower.substring(0, indexLower.length() - '-cloned'.length());
+}
+
 boolean useSystemTenant = false;
 if (params.forceDefaultTenant != true) {
     def itemTypeLower = ctx._source.itemType != null ? ctx._source.itemType.toLowerCase(Locale.ROOT) : '';
     // Shipped ids are grouped by item type, so other types (profiles, events...) skip the lookup
     def shippedIds = params.shippedDefinitionIds != null ? params.shippedDefinitionIds.get(itemTypeLower) : null;
-    if (itemTypeLower == 'geonameentry' || (ctx._index != null && ctx._index.toLowerCase(Locale.ROOT).endsWith('-geonameentry'))) {
+    if (itemTypeLower == 'geonameentry' || indexLower.endsWith('-geonameentry')) {
         useSystemTenant = true;
     } else if (shippedIds != null) {
         def rawId = ctx._source.itemId != null ? ctx._source.itemId : ctx._id;
@@ -49,7 +55,7 @@ if (ctx._id != null) {
     // Skip if already has tenant prefix
     if (!ctx._id.startsWith(params.tenantId + '_') && !ctx._id.startsWith(params.systemTenantId + '_')) {
         def baseId = ctx._id;
-        if (ctx._index != null && ctx._index.toLowerCase(Locale.ROOT).endsWith('-systemitems')) {
+        if (indexLower.endsWith('-systemitems')) {
             // For system items the document ID is tenantId_itemId_itemtype (see getDocumentIDForItemType).
             // 3.0 document IDs already carry the lower-case _itemtype suffix, so only add it when missing.
             def itemType = ctx._source.itemType != null ? ctx._source.itemType.toLowerCase(Locale.ROOT) : null;
@@ -69,7 +75,6 @@ if (ctx._id != null) {
 // systemitems must get audit here: UNOMI-997 puts shipped defs under the system
 // tenant before Unomi starts, so bundle deploy treats them as updates and would
 // otherwise leave createdBy unset on docs that never had it.
-def indexLower = ctx._index != null ? ctx._index.toLowerCase(Locale.ROOT) : '';
 if (!indexLower.endsWith('-geonameentry') && !indexLower.endsWith('-geonames')) {
     // Handle creation date based on item type
     if (ctx._source.creationDate == null) {