Support reading the AlphaTheta OneLibrary format

CMakeLists.txt

@@ -82,6 +82,7 @@ add_library(
     include/djinterop/engine/v3/track_table.hpp
     include/djinterop/exceptions.hpp
     include/djinterop/musical_key.hpp
+    include/djinterop/onelibrary/onelibrary.hpp
     include/djinterop/pad_color.hpp
     include/djinterop/performance_data.hpp
     include/djinterop/playlist.hpp
@@ -203,6 +204,22 @@ add_library(
     src/djinterop/impl/playlist_impl.hpp
     src/djinterop/impl/track_impl.cpp
     src/djinterop/impl/track_impl.hpp
+    src/djinterop/onelibrary/onelibrary.cpp
+    src/djinterop/onelibrary/onelibrary_context.hpp
+    src/djinterop/onelibrary/v1/content_table.cpp
+    src/djinterop/onelibrary/v1/crate_impl.cpp
+    src/djinterop/onelibrary/v1/crate_impl.hpp
+    src/djinterop/onelibrary/v1/database_impl.cpp
+    src/djinterop/onelibrary/v1/database_impl.hpp
+    src/djinterop/onelibrary/v1/library.cpp
+    src/djinterop/onelibrary/v1/playlist_impl.cpp
+    src/djinterop/onelibrary/v1/playlist_impl.hpp
+    src/djinterop/onelibrary/v1/playlist_table.cpp
+    src/djinterop/onelibrary/v1/property_table.cpp
+    src/djinterop/onelibrary/v1/track_conversion.cpp
+    src/djinterop/onelibrary/v1/track_conversion.hpp
+    src/djinterop/onelibrary/v1/track_impl.cpp
+    src/djinterop/onelibrary/v1/track_impl.hpp
     src/djinterop/playlist.cpp
     src/djinterop/track.cpp
     src/djinterop/util/chrono.cpp
@@ -211,9 +228,19 @@ add_library(
     src/djinterop/util/filesystem.hpp
     src/djinterop/util/random.cpp
     src/djinterop/util/random.hpp
+    src/djinterop/util/sqlcipher.hpp
+    src/djinterop/util/sqlite_query.hpp
     src/djinterop/util/sqlite_transaction.hpp
 )
 
+# Encrypted OneLibrary databases are opened with SQLCipher, where the build has
+# it.  Otherwise, opening one fails with an explanation.
+if(EXPERIMENTAL_ENABLE_SQLCIPHER)
+    target_sources(DjInterop PRIVATE src/djinterop/util/sqlcipher.cpp)
+else()
+    target_sources(DjInterop PRIVATE src/djinterop/util/sqlcipher_unsupported.cpp)
+endif()
+
 set_target_properties(DjInterop PROPERTIES
     OUTPUT_NAME "djinterop"
     VERSION ${PROJECT_VERSION}
@@ -238,7 +265,7 @@ target_include_directories(
         DjInterop PUBLIC
         $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/include>
         $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
-        $<INSTALL_INTERFACE:${DJINTEROP_INSTALL_INCLUDEDIR}>)
+        $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>)
 
 # Always rely on system installation of zlib.
 set(ZLIB_MIN_VERSION 1.2.8)
@@ -318,6 +345,14 @@ endif()
 set_target_properties(DjInterop PROPERTIES C_VISIBILITY_PRESET hidden)
 set_target_properties(DjInterop PROPERTIES CXX_VISIBILITY_PRESET hidden)
 
+# A static DjInterop does not carry the SQLCipher library inside it, so that
+# library is installed beside it for consumers to link as well.
+if(EXPERIMENTAL_ENABLE_SQLCIPHER)
+    install(TARGETS sqlcipher
+            EXPORT DjInteropTargets
+            ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}")
+endif()
+
 install(TARGETS DjInterop
         EXPORT DjInteropTargets
         ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"
@@ -370,6 +405,17 @@ install(FILES
     include/djinterop/engine/v3/track_data_blob.hpp
     include/djinterop/engine/v3/track_table.hpp
     DESTINATION "${DJINTEROP_INSTALL_INCLUDEDIR}/engine/v3")
+install(FILES
+    include/djinterop/onelibrary/onelibrary.hpp
+    DESTINATION "${DJINTEROP_INSTALL_INCLUDEDIR}/onelibrary")
+
+install(
+    FILES
+    include/djinterop/onelibrary/v1/content_table.hpp
+    include/djinterop/onelibrary/v1/library.hpp
+    include/djinterop/onelibrary/v1/playlist_table.hpp
+    include/djinterop/onelibrary/v1/property_table.hpp
+    DESTINATION "${DJINTEROP_INSTALL_INCLUDEDIR}/onelibrary/v1")
 
 
 if (UNIX)
@@ -420,6 +466,7 @@ if (BUILD_EXAMPLES)
 
     add_djinterop_example(engine_prime)
     add_djinterop_example(engine_library_v2_low_level)
+    add_djinterop_example(onelibrary)
 endif()
 
 # Unit tests.
@@ -465,6 +512,34 @@ if (Boost_FOUND AND Boost_filesystem_FOUND AND Boost_system_FOUND)
     add_djinterop_test(engine/v3/ performance_data_table_test)
     add_djinterop_test(engine/v3/ track_table_test)
 
+    # The OneLibrary tests talk to SQLite directly to build their fixtures, and
+    # the library's own copy is hidden from them, so they link the SQLCipher
+    # static library themselves.  For the same reason, they compile the
+    # internal sources they test into themselves rather than linking.
+    if(EXPERIMENTAL_ENABLE_SQLCIPHER)
+        add_djinterop_test(onelibrary/ content_table_test)
+        target_sources(onelibrary_content_table_test PRIVATE
+                src/djinterop/onelibrary/v1/content_table.cpp
+                src/djinterop/onelibrary/v1/track_conversion.cpp)
+
+        add_djinterop_test(onelibrary/ database_test)
+
+        add_djinterop_test(onelibrary/ playlist_table_test)
+        target_sources(onelibrary_playlist_table_test PRIVATE
+                src/djinterop/onelibrary/v1/playlist_table.cpp)
+
+        foreach(test_name content_table_test database_test playlist_table_test)
+            target_link_libraries(onelibrary_${test_name} PRIVATE sqlcipher)
+            # The library sources compiled in need the headers they include.
+            target_include_directories(onelibrary_${test_name} PRIVATE SYSTEM
+                    $<TARGET_PROPERTY:DjInterop,INCLUDE_DIRECTORIES>)
+        endforeach()
+    else()
+        message(
+            STATUS
+            "OneLibrary tests not available, as reading the format needs "
+            "EXPERIMENTAL_ENABLE_SQLCIPHER")
+    endif()
 else()
     message(STATUS
             "Unit tests not available, as the Boost.Filesystem and Boost.System "

DjInteropConfig.cmake.in

@@ -14,6 +14,9 @@ endif()
 if(DJINTEROP_SYSTEM_SQLITE)
   find_dependency(SQLite3)
 endif()
+if(DJINTEROP_EXPERIMENTAL_ENABLE_SQLCIPHER)
+  find_dependency(OpenSSL)
+endif()
 
 include("${CMAKE_CURRENT_LIST_DIR}/DjInteropTargets.cmake")
 

GUIDE.md

@@ -73,9 +73,57 @@ As such, in order to create a new library or load an existing library with the
 intention of operating on it using the high-level API, it is always necessary
 to start with format-specific functions to do so:
 
-| Library Type | Include Path                             |
-|--------------|------------------------------------------|
-| Engine       | `#include <djinterop/engine/engine.hpp>` |
+| Library Type | Include Path                                     |
+|--------------|--------------------------------------------------|
+| Engine       | `#include <djinterop/engine/engine.hpp>`         |
+| OneLibrary   | `#include <djinterop/onelibrary/onelibrary.hpp>` |
+
+OneLibrary
+----------
+
+The AlphaTheta OneLibrary format, also documented as Device Library Plus, is
+the successor to the DeviceSQL `export.pdb` library that rekordbox wrote to USB
+media.  A database is loaded by way of `onelibrary::load_database()`, given
+either the root directory of a device or the database file itself.  A number of
+aspects of the format are worth noting:
+
+* Support is currently read-only, and everything that would change a database
+  throws `djinterop::unsupported_operation`.
+* Beat grids, waveforms, hot cues and loops are not held in the database.
+  rekordbox leaves them in the ANLZ files that `content.analysisDataFilePath`
+  points at, and does not populate the `cue` table on export.  Those accessors
+  therefore return nothing rather than throwing.  A caller that reads ANLZ
+  files itself reaches that path through the low-level API, described below.
+* The low-level API also reaches the two other things a device carries that
+  the format-agnostic interface has nowhere to put: the musical key in the
+  notation rekordbox wrote, which may be Camelot and which `track::key()`
+  cannot represent, and the colour the DJ marked a track with, numbered as
+  `export.pdb` numbers them.
+* The format has a single tree that serves as both playlists and crates, so
+  `playlists_and_crates_are_distinct` is false and the two views show the same
+  rows.
+* The database is encrypted with SQLCipher, so reading one needs libdjinterop
+  built with `-DEXPERIMENTAL_ENABLE_SQLCIPHER=ON -DSYSTEM_SQLITE=OFF`, and
+  OpenSSL.  Without it, `load_database()` throws
+  `djinterop::unsupported_database`.
+* rekordbox writes the database in write-ahead-logged mode, and SQLite cannot
+  read one of those without writing beside it, so reading a device creates
+  `-shm` and `-wal` files next to `exportLibrary.db` if they are missing.  The
+  device therefore has to be writable.
+
+### OneLibrary low-level API
+
+The low-level API is in `<djinterop/onelibrary/v1/library.hpp>`, and exposes
+the tables as the device holds them, translating no further than resolving a
+lookup reference to the text behind it.  A device is loaded as an
+`onelibrary::v1::library`, whose `content()`, `playlist()` and `property()`
+give the tables, and whose `database()` gives the same database that
+`load_database()` would have.
+
+The `v1` is the schema those tables describe.  A device records it as
+`property.dbVersion`, and every export seen so far reports `1000`.  A schema
+not compatible with this one gets a namespace of its own, as the Engine
+formats do.
 
 
 Stable API/ABI

README.md

@@ -8,7 +8,8 @@ State of Support
 ================
 
 The library is currently in development, and not all features are implemented
-yet.  It currently supports only the Engine Library format.
+yet.  It supports the Engine Library format, and can read the AlphaTheta
+OneLibrary format.
 
 What is supported:
 
@@ -27,11 +28,22 @@ The library supports the following firmware and application versions:
     SC6000/M) may work, but this is currently untested.
 * Engine DJ Desktop (aka Engine Prime) from 1.0.1 to 4.3.0.
 
+The library also reads the AlphaTheta OneLibrary format, which rekordbox 7
+writes to USB media as `PIONEER/rekordbox/exportLibrary.db`, and which players
+from the CDJ-3000X, XDJ-AZ, OPUS-QUAD and OMNIS-DUO onwards read, as does the
+CDJ-3000 from firmware 3.15.  Track metadata, playlists and crates can be read.
+Writing is not supported yet, and beat grids, waveforms, hot cues and loops are
+not held in the database at all, as rekordbox leaves them in the ANLZ files
+beside it.  The format is also documented under the name Device Library Plus.
+The database is encrypted with SQLCipher, so reading it needs the library built
+with `-DEXPERIMENTAL_ENABLE_SQLCIPHER=ON -DSYSTEM_SQLITE=OFF`, and OpenSSL.
+
 What is not supported (yet):
 
 * Album art
 * Play history
-* DJ record libraries in formats other than Engine Prime
+* Writing OneLibrary databases
+* DJ record libraries in formats other than Engine Prime and OneLibrary
 
 How Do I Use It?
 ================

example/README.md

@@ -4,9 +4,29 @@ Overview
 This directory contains small example applications that illustrate the use
 of `libdjinterop`.
 
-This application can be minimally compiled in isolation with an invocation
-similar to the below (adjust for your favourite compiler as appropriate):
+| Example                       | What it does                                                        |
+|-------------------------------|---------------------------------------------------------------------|
+| `engine_prime`                | Writes a track, a crate and a playlist to an Engine Prime library.  |
+| `engine_library_v2_low_level` | Uses the low-level Engine v2 API to work with tables directly.      |
+| `onelibrary`                  | Prints the tracks and playlists of an AlphaTheta OneLibrary device. |
+
+They are built by the project itself, as `example_engine_prime` and so on,
+when it is configured with `-DBUILD_EXAMPLES=ON`.
+
+To build one of them against an installed `libdjinterop` instead, as a
+starting point for a program of your own:
+
+```shell
+g++ -std=c++20 `pkg-config --cflags djinterop` onelibrary.cpp `pkg-config --libs djinterop` -o onelibrary
+```
+
+The library needs a C++20 compiler, and its headers refuse to compile under
+any older standard.
+
+`onelibrary` takes the device to read as its argument, either the root
+directory of a device or the `exportLibrary.db` file itself, and optionally a
+passphrase:
 
 ```shell
-g++ -std=c++17 `pkg-config --cflags djinterop` engine_prime.cpp `pkg-config --libs djinterop`
-```
+./onelibrary /Volumes/MYUSB
+```

example/onelibrary.cpp

@@ -0,0 +1,103 @@
+/*
+    This file is part of libdjinterop.
+
+    libdjinterop is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    libdjinterop is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with libdjinterop.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+// Print the contents of an AlphaTheta OneLibrary device.
+//
+//     onelibrary /Volumes/MYUSB [passphrase]
+//
+// The first argument is the root of the device -- the directory holding
+// `PIONEER` -- or the `exportLibrary.db` file itself.  The second is the
+// passphrase, which defaults to the one rekordbox uses.
+
+#include <iostream>
+#include <string>
+
+#include <djinterop/djinterop.hpp>
+
+namespace
+{
+void print_playlist(const djinterop::playlist& pl, int depth)
+{
+    const std::string indent(static_cast<size_t>(depth) * 2, ' ');
+    std::cout << indent << "- " << pl.name() << " (" << pl.tracks().size()
+              << " tracks)\n";
+
+    for (auto&& child : pl.children())
+        print_playlist(child, depth + 1);
+}
+
+}  // anonymous namespace
+
+int main(int argc, char** argv)
+{
+    if (argc < 2)
+    {
+        std::cerr << "usage: onelibrary <device> [passphrase]\n";
+        return 2;
+    }
+
+    const std::string device = argv[1];
+    const std::string passphrase =
+        argc > 2 ? argv[2] : djinterop::onelibrary::default_passphrase;
+
+    if (!djinterop::onelibrary::database_exists(device))
+    {
+        std::cerr << "No OneLibrary database found in " << device << "\n";
+        return 1;
+    }
+
+    try
+    {
+        auto db = djinterop::onelibrary::load_database(device, passphrase);
+
+        std::cout << "Format:  " << db.version_name() << "\n"
+                  << "Library: " << db.uuid() << "\n\n";
+
+        std::cout << "Tracks\n------\n";
+        for (auto&& track : db.tracks())
+        {
+            // A snapshot reads the whole track at once, where each accessor
+            // would query the database again.
+            const auto snapshot = track.snapshot();
+            std::cout << track.id() << ". "
+                      << snapshot.title.value_or("(untitled)") << " - "
+                      << snapshot.artist.value_or("(unknown artist)");
+
+            if (snapshot.bpm)
+                std::cout << " [" << *snapshot.bpm << " BPM]";
+
+            if (snapshot.key)
+                std::cout << " [" << *snapshot.key << "]";
+
+            std::cout << "\n    " << snapshot.relative_path.value_or("")
+                      << "\n";
+        }
+
+        std::cout << "\nPlaylists\n---------\n";
+        for (auto&& pl : db.root_playlists())
+            print_playlist(pl, 0);
+    }
+    catch (const djinterop::unsupported_database& e)
+    {
+        // The most likely cause is a passphrase that does not open the
+        // database, which happens if rekordbox has changed it.
+        std::cerr << "Could not read the library: " << e.what() << "\n";
+        return 1;
+    }
+
+    return 0;
+}

ext/sqlcipher/CMakeLists.txt

@@ -27,7 +27,7 @@ target_compile_definitions(
 
 target_include_directories(
     sqlcipher PUBLIC
-    ${CMAKE_CURRENT_SOURCE_DIR})
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>)
 
 # SQLCipher needs OpenSSL for AES-256-CBC, PBKDF2-HMAC-SHA512
 find_package(OpenSSL REQUIRED)

include/djinterop/djinterop.hpp

@@ -27,6 +27,7 @@
 #include <djinterop/engine/engine.hpp>
 #include <djinterop/exceptions.hpp>
 #include <djinterop/musical_key.hpp>
+#include <djinterop/onelibrary/onelibrary.hpp>
 #include <djinterop/pad_color.hpp>
 #include <djinterop/performance_data.hpp>
 #include <djinterop/playlist.hpp>